root/tools/crm_mon.c

/* */

DEFINITIONS

1. all_includes
2. default_includes
3. find_section_bit
4. apply_exclude
5. apply_include
6. apply_include_exclude
7. user_include_exclude_cb
8. include_exclude_cb
9. as_cgi_cb
10. as_html_cb
11. as_simple_cb
12. as_xml_cb
13. fence_history_cb
14. group_by_node_cb
15. hide_headers_cb
16. inactive_resources_cb
17. no_curses_cb
18. print_brief_cb
19. print_detail_cb
20. print_timing_cb
21. reconnect_cb
22. show_attributes_cb
23. show_bans_cb
24. show_failcounts_cb
25. show_operations_cb
26. show_tickets_cb
27. use_cib_file_cb
28. reconnect_after_timeout
29. mon_cib_connection_destroy
30. mon_shutdown
31. mon_winresize
32. fencing_connect
33. cib_connect
34. set_fencing_options
35. pacemakerd_event_cb
36. pacemakerd_status
37. get_option_desc
38. detect_user_input
39. avoid_zombies
40. build_arg_context
41. add_output_args
42. reconcile_output_format
43. handle_connection_failures
44. one_shot
45. main
46. print_simple_status
47. send_custom_trap
48. handle_rsc_op
49. mon_trigger_refresh
50. crm_diff_update_v2
51. crm_diff_update_v1
52. crm_diff_update
53. get_fencing_history
54. mon_refresh_display
55. mon_st_callback_event
56. refresh_after_event
57. mon_st_callback_display
58. clean_up_cib_connection
59. clean_up_fencing_connection
60. clean_up

   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 
  14 #include <crm/crm.h>
  15 
  16 #include <stdio.h>
  17 #include <sys/types.h>
  18 #include <sys/stat.h>
  19 #include <unistd.h>
  20 
  21 #include <stdlib.h>
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <libgen.h>
  25 #include <signal.h>
  26 #include <sys/utsname.h>
  27 
  28 #include <crm/msg_xml.h>
  29 #include <crm/services.h>
  30 #include <crm/lrmd.h>
  31 #include <crm/common/cmdline_internal.h>
  32 #include <crm/common/internal.h>  // pcmk__ends_with_ext()
  33 #include <crm/common/ipc.h>
  34 #include <crm/common/mainloop.h>
  35 #include <crm/common/output.h>
  36 #include <crm/common/output_internal.h>
  37 #include <crm/common/util.h>
  38 #include <crm/common/xml.h>
  39 #include <crm/common/xml_internal.h>
  40 
  41 #include <crm/cib/internal.h>
  42 #include <crm/pengine/status.h>
  43 #include <crm/pengine/internal.h>
  44 #include <pacemaker-internal.h>
  45 #include <crm/stonith-ng.h>
  46 #include <crm/fencing/internal.h>
  47 
  48 #include "crm_mon.h"
  49 
  50 #define SUMMARY "Provides a summary of cluster's current state.\n\n" \
  51                 "Outputs varying levels of detail in a number of different formats."
  52 
  53 /*
  54  * Definitions indicating which items to print
  55  */
  56 
  57 static unsigned int show;
  58 static unsigned int show_opts = pcmk_show_pending;
  59 
  60 /*
  61  * Definitions indicating how to output
  62  */
  63 
  64 static mon_output_format_t output_format = mon_output_unset;
  65 
  66 /* other globals */
  67 static GIOChannel *io_channel = NULL;
  68 static GMainLoop *mainloop = NULL;
  69 static guint reconnect_timer = 0;
  70 static mainloop_timer_t *refresh_timer = NULL;
  71 static pe_working_set_t *mon_data_set = NULL;
  72 
  73 static cib_t *cib = NULL;
  74 static stonith_t *st = NULL;
  75 static xmlNode *current_cib = NULL;
  76 
  77 static GError *error = NULL;
  78 static pcmk__common_args_t *args = NULL;
  79 static pcmk__output_t *out = NULL;
  80 static GOptionContext *context = NULL;
  81 static gchar **processed_args = NULL;
  82 
  83 static time_t last_refresh = 0;
  84 volatile crm_trigger_t *refresh_trigger = NULL;
  85 
  86 static gboolean fence_history = FALSE;
  87 static gboolean has_warnings = FALSE;
  88 static gboolean on_remote_node = FALSE;
  89 static gboolean use_cib_native = FALSE;
  90 
  91 int interactive_fence_level = 0;
  92 
  93 static pcmk__supported_format_t formats[] = {
  94 #if CURSES_ENABLED
  95     CRM_MON_SUPPORTED_FORMAT_CURSES,
  96 #endif
  97     PCMK__SUPPORTED_FORMAT_HTML,
  98     PCMK__SUPPORTED_FORMAT_NONE,
  99     PCMK__SUPPORTED_FORMAT_TEXT,
 100     PCMK__SUPPORTED_FORMAT_XML,
 101     { NULL, NULL, NULL }
 102 };
 103 
 104 /* Define exit codes for monitoring-compatible output
 105  * For nagios plugins, the possibilities are
 106  * OK=0, WARN=1, CRIT=2, and UNKNOWN=3
 107  */
 108 #define MON_STATUS_WARN    CRM_EX_ERROR
 109 #define MON_STATUS_CRIT    CRM_EX_INVALID_PARAM
 110 #define MON_STATUS_UNKNOWN CRM_EX_UNIMPLEMENT_FEATURE
 111 
 112 #define RECONNECT_MSECS 5000
 113 
 114 struct {
 115     guint reconnect_ms;
 116     gboolean daemonize;
 117     gboolean fence_connect;
 118     gboolean one_shot;
 119     gboolean print_pending;
 120     gboolean show_bans;
 121     gboolean watch_fencing;
 122     char *pid_file;
 123     char *external_agent;
 124     char *external_recipient;
 125     char *neg_location_prefix;
 126     char *only_node;
 127     char *only_rsc;
 128     GSList *user_includes_excludes;
 129     GSList *includes_excludes;
 130 } options = {
 131     .fence_connect = TRUE,
 132     .reconnect_ms = RECONNECT_MSECS
 133 };
 134 
 135 static void clean_up_cib_connection(void);
 136 static void clean_up_fencing_connection(void);
 137 static crm_exit_t clean_up(crm_exit_t exit_code);
 138 static void crm_diff_update(const char *event, xmlNode * msg);
 139 static void handle_connection_failures(int rc);
 140 static int mon_refresh_display(gpointer user_data);
 141 static int cib_connect(gboolean full);
 142 static int fencing_connect(void);
 143 static int pacemakerd_status(void);
 144 static void mon_st_callback_event(stonith_t * st, stonith_event_t * e);
 145 static void mon_st_callback_display(stonith_t * st, stonith_event_t * e);
 146 static void refresh_after_event(gboolean data_updated, gboolean enforce);
 147 
 148 static unsigned int
 149 all_includes(mon_output_format_t fmt) {
     /*  */
 150     if (fmt == mon_output_monitor || fmt == mon_output_plain || fmt == mon_output_console) {
 151         return ~pcmk_section_options;
 152     } else {
 153         return pcmk_section_all;
 154     }
 155 }
 156 
 157 static unsigned int
 158 default_includes(mon_output_format_t fmt) {
     /*  */
 159     switch (fmt) {
 160         case mon_output_monitor:
 161         case mon_output_plain:
 162         case mon_output_console:
 163             return pcmk_section_summary | pcmk_section_nodes | pcmk_section_resources |
 164                    pcmk_section_failures;
 165 
 166         case mon_output_xml:
 167         case mon_output_legacy_xml:
 168             return all_includes(fmt);
 169 
 170         case mon_output_html:
 171         case mon_output_cgi:
 172             return pcmk_section_summary | pcmk_section_nodes | pcmk_section_resources |
 173                    pcmk_section_failures;
 174 
 175         default:
 176             return 0;
 177     }
 178 }
 179 
 180 struct {
 181     const char *name;
 182     unsigned int bit;
 183 } sections[] = {
 184     { "attributes", pcmk_section_attributes },
 185     { "bans", pcmk_section_bans },
 186     { "counts", pcmk_section_counts },
 187     { "dc", pcmk_section_dc },
 188     { "failcounts", pcmk_section_failcounts },
 189     { "failures", pcmk_section_failures },
 190     { "fencing", pcmk_section_fencing_all },
 191     { "fencing-failed", pcmk_section_fence_failed },
 192     { "fencing-pending", pcmk_section_fence_pending },
 193     { "fencing-succeeded", pcmk_section_fence_worked },
 194     { "maint-mode", pcmk_section_maint_mode },
 195     { "nodes", pcmk_section_nodes },
 196     { "operations", pcmk_section_operations },
 197     { "options", pcmk_section_options },
 198     { "resources", pcmk_section_resources },
 199     { "stack", pcmk_section_stack },
 200     { "summary", pcmk_section_summary },
 201     { "tickets", pcmk_section_tickets },
 202     { "times", pcmk_section_times },
 203     { NULL }
 204 };
 205 
 206 static unsigned int
 207 find_section_bit(const char *name) {
     /*  */
 208     for (int i = 0; sections[i].name != NULL; i++) {
 209         if (pcmk__str_eq(sections[i].name, name, pcmk__str_casei)) {
 210             return sections[i].bit;
 211         }
 212     }
 213 
 214     return 0;
 215 }
 216 
 217 static gboolean
 218 apply_exclude(const gchar *excludes, GError **error) {
     /*  */
 219     char **parts = NULL;
 220     gboolean result = TRUE;
 221 
 222     parts = g_strsplit(excludes, ",", 0);
 223     for (char **s = parts; *s != NULL; s++) {
 224         unsigned int bit = find_section_bit(*s);
 225 
 226         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 227             show = 0;
 228         } else if (pcmk__str_eq(*s, "none", pcmk__str_none)) {
 229             show = all_includes(output_format);
 230         } else if (bit != 0) {
 231             show &= ~bit;
 232         } else {
 233             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 234                         "--exclude options: all, attributes, bans, counts, dc, "
 235                         "failcounts, failures, fencing, fencing-failed, "
 236                         "fencing-pending, fencing-succeeded, maint-mode, nodes, "
 237                         "none, operations, options, resources, stack, summary, "
 238                         "tickets, times");
 239             result = FALSE;
 240             break;
 241         }
 242     }
 243     g_strfreev(parts);
 244     return result;
 245 }
 246 
 247 static gboolean
 248 apply_include(const gchar *includes, GError **error) {
     /*  */
 249     char **parts = NULL;
 250     gboolean result = TRUE;
 251 
 252     parts = g_strsplit(includes, ",", 0);
 253     for (char **s = parts; *s != NULL; s++) {
 254         unsigned int bit = find_section_bit(*s);
 255 
 256         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 257             show = all_includes(output_format);
 258         } else if (pcmk__starts_with(*s, "bans")) {
 259             show |= pcmk_section_bans;
 260             if (options.neg_location_prefix != NULL) {
 261                 free(options.neg_location_prefix);
 262                 options.neg_location_prefix = NULL;
 263             }
 264 
 265             if (strlen(*s) > 4 && (*s)[4] == ':') {
 266                 options.neg_location_prefix = strdup(*s+5);
 267             }
 268         } else if (pcmk__str_any_of(*s, "default", "defaults", NULL)) {
 269             show |= default_includes(output_format);
 270         } else if (pcmk__str_eq(*s, "none", pcmk__str_none)) {
 271             show = 0;
 272         } else if (bit != 0) {
 273             show |= bit;
 274         } else {
 275             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 276                         "--include options: all, attributes, bans[:PREFIX], counts, dc, "
 277                         "default, failcounts, failures, fencing, fencing-failed, "
 278                         "fencing-pending, fencing-succeeded, maint-mode, nodes, none, "
 279                         "operations, options, resources, stack, summary, tickets, times");
 280             result = FALSE;
 281             break;
 282         }
 283     }
 284     g_strfreev(parts);
 285     return result;
 286 }
 287 
 288 static gboolean
 289 apply_include_exclude(GSList *lst, GError **error) {
     /*  */
 290     gboolean rc = TRUE;
 291     GSList *node = lst;
 292 
 293     while (node != NULL) {
 294         char *s = node->data;
 295 
 296         if (pcmk__starts_with(s, "--include=")) {
 297             rc = apply_include(s+10, error);
 298         } else if (pcmk__starts_with(s, "-I=")) {
 299             rc = apply_include(s+3, error);
 300         } else if (pcmk__starts_with(s, "--exclude=")) {
 301             rc = apply_exclude(s+10, error);
 302         } else if (pcmk__starts_with(s, "-U=")) {
 303             rc = apply_exclude(s+3, error);
 304         }
 305 
 306         if (rc != TRUE) {
 307             break;
 308         }
 309 
 310         node = node->next;
 311     }
 312 
 313     return rc;
 314 }
 315 
 316 static gboolean
 317 user_include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 318     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 319 
 320     options.user_includes_excludes = g_slist_append(options.user_includes_excludes, s);
 321     return TRUE;
 322 }
 323 
 324 static gboolean
 325 include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 326     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 327 
 328     options.includes_excludes = g_slist_append(options.includes_excludes, s);
 329     return TRUE;
 330 }
 331 
 332 static gboolean
 333 as_cgi_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 334     if (args->output_ty != NULL) {
 335         free(args->output_ty);
 336     }
 337 
 338     args->output_ty = strdup("html");
 339     output_format = mon_output_cgi;
 340     options.one_shot = TRUE;
 341     return TRUE;
 342 }
 343 
 344 static gboolean
 345 as_html_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 346     if (args->output_ty != NULL) {
 347         free(args->output_ty);
 348     }
 349 
 350     if (args->output_dest != NULL) {
 351         free(args->output_dest);
 352         args->output_dest = NULL;
 353     }
 354 
 355     if (optarg != NULL) {
 356         args->output_dest = strdup(optarg);
 357     }
 358 
 359     args->output_ty = strdup("html");
 360     output_format = mon_output_html;
 361     umask(S_IWGRP | S_IWOTH);
 362     return TRUE;
 363 }
 364 
 365 static gboolean
 366 as_simple_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 367     if (args->output_ty != NULL) {
 368         free(args->output_ty);
 369     }
 370 
 371     args->output_ty = strdup("text");
 372     output_format = mon_output_monitor;
 373     options.one_shot = TRUE;
 374     return TRUE;
 375 }
 376 
 377 static gboolean
 378 as_xml_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 379     if (args->output_ty != NULL) {
 380         free(args->output_ty);
 381     }
 382 
 383     args->output_ty = strdup("xml");
 384     output_format = mon_output_legacy_xml;
 385     return TRUE;
 386 }
 387 
 388 static gboolean
 389 fence_history_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 390     if (optarg == NULL) {
 391         interactive_fence_level = 2;
 392     } else {
 393         pcmk__scan_min_int(optarg, &interactive_fence_level, 0);
 394     }
 395 
 396     switch (interactive_fence_level) {
 397         case 3:
 398             options.fence_connect = TRUE;
 399             fence_history = TRUE;
 400             return include_exclude_cb("--include", "fencing", data, err);
 401 
 402         case 2:
 403             options.fence_connect = TRUE;
 404             fence_history = TRUE;
 405             return include_exclude_cb("--include", "fencing", data, err);
 406 
 407         case 1:
 408             options.fence_connect = TRUE;
 409             fence_history = TRUE;
 410             return include_exclude_cb("--include", "fencing-failed,fencing-pending", data, err);
 411 
 412         case 0:
 413             options.fence_connect = FALSE;
 414             fence_history = FALSE;
 415             return include_exclude_cb("--exclude", "fencing", data, err);
 416 
 417         default:
 418             g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Fence history must be 0-3");
 419             return FALSE;
 420     }
 421 }
 422 
 423 static gboolean
 424 group_by_node_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 425     show_opts |= pcmk_show_rscs_by_node;
 426     return TRUE;
 427 }
 428 
 429 static gboolean
 430 hide_headers_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 431     return user_include_exclude_cb("--exclude", "summary", data, err);
 432 }
 433 
 434 static gboolean
 435 inactive_resources_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 436     show_opts |= pcmk_show_inactive_rscs;
 437     return TRUE;
 438 }
 439 
 440 static gboolean
 441 no_curses_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 442     output_format = mon_output_plain;
 443     return TRUE;
 444 }
 445 
 446 static gboolean
 447 print_brief_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 448     show_opts |= pcmk_show_brief;
 449     return TRUE;
 450 }
 451 
 452 static gboolean
 453 print_detail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 454     show_opts |= pcmk_show_details;
 455     return TRUE;
 456 }
 457 
 458 static gboolean
 459 print_timing_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 460     show_opts |= pcmk_show_timing;
 461     return user_include_exclude_cb("--include", "operations", data, err);
 462 }
 463 
 464 static gboolean
 465 reconnect_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 466     int rc = crm_get_msec(optarg);
 467 
 468     if (rc == -1) {
 469         g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid value for -i: %s", optarg);
 470         return FALSE;
 471     } else {
 472         options.reconnect_ms = crm_parse_interval_spec(optarg);
 473     }
 474 
 475     return TRUE;
 476 }
 477 
 478 static gboolean
 479 show_attributes_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 480     return user_include_exclude_cb("--include", "attributes", data, err);
 481 }
 482 
 483 static gboolean
 484 show_bans_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 485     if (optarg != NULL) {
 486         char *s = crm_strdup_printf("bans:%s", optarg);
 487         gboolean rc = user_include_exclude_cb("--include", s, data, err);
 488         free(s);
 489         return rc;
 490     } else {
 491         return user_include_exclude_cb("--include", "bans", data, err);
 492     }
 493 }
 494 
 495 static gboolean
 496 show_failcounts_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 497     return user_include_exclude_cb("--include", "failcounts", data, err);
 498 }
 499 
 500 static gboolean
 501 show_operations_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 502     return user_include_exclude_cb("--include", "failcounts,operations", data, err);
 503 }
 504 
 505 static gboolean
 506 show_tickets_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 507     return user_include_exclude_cb("--include", "tickets", data, err);
 508 }
 509 
 510 static gboolean
 511 use_cib_file_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 512     setenv("CIB_file", optarg, 1);
 513     options.one_shot = TRUE;
 514     return TRUE;
 515 }
 516 
 517 #define INDENT "                                    "
 518 
 519 /* *INDENT-OFF* */
 520 static GOptionEntry addl_entries[] = {
 521     { "interval", 'i', 0, G_OPTION_ARG_CALLBACK, reconnect_cb,
 522       "Update frequency (default is 5 seconds)",
 523       "TIMESPEC" },
 524 
 525     { "one-shot", '1', 0, G_OPTION_ARG_NONE, &options.one_shot,
 526       "Display the cluster status once on the console and exit",
 527       NULL },
 528 
 529     { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &options.daemonize,
 530       "Run in the background as a daemon.\n"
 531       INDENT "Requires at least one of --output-to and --external-agent.",
 532       NULL },
 533 
 534     { "pid-file", 'p', 0, G_OPTION_ARG_FILENAME, &options.pid_file,
 535       "(Advanced) Daemon pid file location",
 536       "FILE" },
 537 
 538     { "external-agent", 'E', 0, G_OPTION_ARG_FILENAME, &options.external_agent,
 539       "A program to run when resource operations take place",
 540       "FILE" },
 541 
 542     { "external-recipient", 'e', 0, G_OPTION_ARG_STRING, &options.external_recipient,
 543       "A recipient for your program (assuming you want the program to send something to someone).",
 544       "RCPT" },
 545 
 546     { "watch-fencing", 'W', 0, G_OPTION_ARG_NONE, &options.watch_fencing,
 547       "Listen for fencing events. For use with --external-agent.",
 548       NULL },
 549 
 550     { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, use_cib_file_cb,
 551       NULL,
 552       NULL },
 553 
 554     { NULL }
 555 };
 556 
 557 static GOptionEntry display_entries[] = {
 558     { "include", 'I', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 559       "A list of sections to include in the output.\n"
 560       INDENT "See `Output Control` help for more information.",
 561       "SECTION(s)" },
 562 
 563     { "exclude", 'U', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 564       "A list of sections to exclude from the output.\n"
 565       INDENT "See `Output Control` help for more information.",
 566       "SECTION(s)" },
 567 
 568     { "node", 0, 0, G_OPTION_ARG_STRING, &options.only_node,
 569       "When displaying information about nodes, show only what's related to the given\n"
 570       INDENT "node, or to all nodes tagged with the given tag",
 571       "NODE" },
 572 
 573     { "resource", 0, 0, G_OPTION_ARG_STRING, &options.only_rsc,
 574       "When displaying information about resources, show only what's related to the given\n"
 575       INDENT "resource, or to all resources tagged with the given tag",
 576       "RSC" },
 577 
 578     { "group-by-node", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, group_by_node_cb,
 579       "Group resources by node",
 580       NULL },
 581 
 582     { "inactive", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, inactive_resources_cb,
 583       "Display inactive resources",
 584       NULL },
 585 
 586     { "failcounts", 'f', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_failcounts_cb,
 587       "Display resource fail counts",
 588       NULL },
 589 
 590     { "operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_operations_cb,
 591       "Display resource operation history",
 592       NULL },
 593 
 594     { "timing-details", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_timing_cb,
 595       "Display resource operation history with timing details",
 596       NULL },
 597 
 598     { "tickets", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_tickets_cb,
 599       "Display cluster tickets",
 600       NULL },
 601 
 602     { "fence-history", 'm', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, fence_history_cb,
 603       "Show fence history:\n"
 604       INDENT "0=off, 1=failures and pending (default without option),\n"
 605       INDENT "2=add successes (default without value for option),\n"
 606       INDENT "3=show full history without reduction to most recent of each flavor",
 607       "LEVEL" },
 608 
 609     { "neg-locations", 'L', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, show_bans_cb,
 610       "Display negative location constraints [optionally filtered by id prefix]",
 611       NULL },
 612 
 613     { "show-node-attributes", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_attributes_cb,
 614       "Display node attributes",
 615       NULL },
 616 
 617     { "hide-headers", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, hide_headers_cb,
 618       "Hide all headers",
 619       NULL },
 620 
 621     { "show-detail", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_detail_cb,
 622       "Show more details (node IDs, individual clone instances)",
 623       NULL },
 624 
 625     { "brief", 'b', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_brief_cb,
 626       "Brief output",
 627       NULL },
 628 
 629     { "pending", 'j', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.print_pending,
 630       "Display pending state if 'record-pending' is enabled",
 631       NULL },
 632 
 633     { "simple-status", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_simple_cb,
 634       "Display the cluster status once as a simple one line output (suitable for nagios)",
 635       NULL },
 636 
 637     { NULL }
 638 };
 639 
 640 static GOptionEntry deprecated_entries[] = {
 641     { "as-html", 'h', G_OPTION_FLAG_FILENAME, G_OPTION_ARG_CALLBACK, as_html_cb,
 642       "Write cluster status to the named HTML file.\n"
 643       INDENT "Use --output-as=html --output-to=FILE instead.",
 644       "FILE" },
 645 
 646     { "as-xml", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_xml_cb,
 647       "Write cluster status as XML to stdout. This will enable one-shot mode.\n"
 648       INDENT "Use --output-as=xml instead.",
 649       NULL },
 650 
 651     { "disable-ncurses", 'N', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, no_curses_cb,
 652       "Disable the use of ncurses.\n"
 653       INDENT "Use --output-as=text instead.",
 654       NULL },
 655 
 656     { "web-cgi", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_cgi_cb,
 657       "Web mode with output suitable for CGI (preselected when run as *.cgi).\n"
 658       INDENT "Use --output-as=html --html-cgi instead.",
 659       NULL },
 660 
 661     { NULL }
 662 };
 663 /* *INDENT-ON* */
 664 
 665 /* Reconnect to the CIB and fencing agent after reconnect_ms has passed.  This sounds
 666  * like it would be more broadly useful, but only ever happens after a disconnect via
 667  * mon_cib_connection_destroy.
 668  */
 669 static gboolean
 670 reconnect_after_timeout(gpointer data)
     /*  */
 671 {
 672 #if CURSES_ENABLED
 673     if (output_format == mon_output_console) {
 674         clear();
 675         refresh();
 676     }
 677 #endif
 678 
 679     out->info(out, "Reconnecting...");
 680     if (pacemakerd_status() == pcmk_rc_ok) {
 681         fencing_connect();
 682         if (cib_connect(TRUE) == pcmk_rc_ok) {
 683             /* trigger redrawing the screen (needs reconnect_timer == 0) */
 684             reconnect_timer = 0;
 685             refresh_after_event(FALSE, TRUE);
 686             return G_SOURCE_REMOVE;
 687         }
 688     }
 689 
 690     reconnect_timer = g_timeout_add(options.reconnect_ms,
 691                                     reconnect_after_timeout, NULL);
 692     return G_SOURCE_REMOVE;
 693 }
 694 
 695 /* Called from various places when we are disconnected from the CIB or from the
 696  * fencing agent.  If the CIB connection is still valid, this function will also
 697  * attempt to sign off and reconnect.
 698  */
 699 static void
 700 mon_cib_connection_destroy(gpointer user_data)
     /*  */
 701 {
 702     out->info(out, "Connection to the cluster-daemons terminated");
 703 
 704     if (refresh_timer != NULL) {
 705         /* we'll trigger a refresh after reconnect */
 706         mainloop_timer_stop(refresh_timer);
 707     }
 708     if (reconnect_timer) {
 709         /* we'll trigger a new reconnect-timeout at the end */
 710         g_source_remove(reconnect_timer);
 711         reconnect_timer = 0;
 712     }
 713     if (st) {
 714         /* the client API won't properly reconnect notifications
 715          * if they are still in the table - so remove them
 716          */
 717         clean_up_fencing_connection();
 718     }
 719     if (cib) {
 720         cib->cmds->signoff(cib);
 721         reconnect_timer = g_timeout_add(options.reconnect_ms,
 722                                         reconnect_after_timeout, NULL);
 723     }
 724     return;
 725 }
 726 
 727 /* Signal handler installed into the mainloop for normal program shutdown */
 728 static void
 729 mon_shutdown(int nsig)
     /*  */
 730 {
 731     clean_up(CRM_EX_OK);
 732 }
 733 
 734 #if CURSES_ENABLED
 735 static volatile sighandler_t ncurses_winch_handler;
 736 
 737 /* Signal handler installed the regular way (not into the main loop) for when
 738  * the screen is resized.  Commonly, this happens when running in an xterm and
 739  * the user changes its size.
 740  */
 741 static void
 742 mon_winresize(int nsig)
     /*  */
 743 {
 744     static int not_done;
 745     int lines = 0, cols = 0;
 746 
 747     if (!not_done++) {
 748         if (ncurses_winch_handler)
 749             /* the original ncurses WINCH signal handler does the
 750              * magic of retrieving the new window size;
 751              * otherwise, we'd have to use ioctl or tgetent */
 752             (*ncurses_winch_handler) (SIGWINCH);
 753         getmaxyx(stdscr, lines, cols);
 754         resizeterm(lines, cols);
 755         /* Alert the mainloop code we'd like the refresh_trigger to run next
 756          * time the mainloop gets around to checking.
 757          */
 758         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
 759     }
 760     not_done--;
 761 }
 762 #endif
 763 
 764 static int
 765 fencing_connect(void)
     /*  */
 766 {
 767     int rc = pcmk_ok;
 768 
 769     if (options.fence_connect && st == NULL) {
 770         st = stonith_api_new();
 771     }
 772 
 773     if (!options.fence_connect || st == NULL || st->state != stonith_disconnected) {
 774         return rc;
 775     }
 776 
 777     rc = st->cmds->connect(st, crm_system_name, NULL);
 778     if (rc == pcmk_ok) {
 779         crm_trace("Setting up stonith callbacks");
 780         if (options.watch_fencing) {
 781             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 782                                             mon_st_callback_event);
 783             st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback_event);
 784         } else {
 785             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 786                                             mon_st_callback_display);
 787             st->cmds->register_notification(st, T_STONITH_NOTIFY_HISTORY, mon_st_callback_display);
 788         }
 789     } else {
 790         clean_up_fencing_connection();
 791     }
 792 
 793     return rc;
 794 }
 795 
 796 static int
 797 cib_connect(gboolean full)
     /*  */
 798 {
 799     int rc = pcmk_rc_ok;
 800 
 801     CRM_CHECK(cib != NULL, return EINVAL);
 802 
 803     if (cib->state == cib_connected_query ||
 804         cib->state == cib_connected_command) {
 805         return rc;
 806     }
 807 
 808     crm_trace("Connecting to the CIB");
 809 
 810     rc = pcmk_legacy2rc(cib->cmds->signon(cib, crm_system_name, cib_query));
 811     if (rc != pcmk_rc_ok) {
 812         out->err(out, "Could not connect to the CIB: %s",
 813                  pcmk_rc_str(rc));
 814         return rc;
 815     }
 816 
 817 #if CURSES_ENABLED
 818     /* just show this if refresh is gonna remove all traces */
 819     if (output_format == mon_output_console) {
 820         out->info(out,"Waiting for CIB ...");
 821     }
 822 #endif
 823 
 824     rc = pcmk_legacy2rc(cib->cmds->query(cib, NULL, &current_cib,
 825                                          cib_scope_local | cib_sync_call));
 826 
 827     if (rc == pcmk_rc_ok && full) {
 828         rc = pcmk_legacy2rc(cib->cmds->set_connection_dnotify(cib,
 829             mon_cib_connection_destroy));
 830         if (rc == EPROTONOSUPPORT) {
 831             out->err(out,
 832                      "Notification setup not supported, won't be "
 833                      "able to reconnect after failure");
 834             if (output_format == mon_output_console) {
 835                 sleep(2);
 836             }
 837             rc = pcmk_rc_ok;
 838         }
 839 
 840         if (rc == pcmk_rc_ok) {
 841             cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY,
 842                                            crm_diff_update);
 843             rc = pcmk_legacy2rc(cib->cmds->add_notify_callback(cib,
 844                                     T_CIB_DIFF_NOTIFY, crm_diff_update));
 845         }
 846 
 847         if (rc != pcmk_rc_ok) {
 848             out->err(out, "Notification setup failed, could not monitor CIB actions");
 849             clean_up_cib_connection();
 850             clean_up_fencing_connection();
 851         }
 852     }
 853     return rc;
 854 }
 855 
 856 /* This is used to set up the fencing options after the interactive UI has been stared.
 857  * fence_history_cb can't be used because it builds up a list of includes/excludes that
 858  * then have to be processed with apply_include_exclude and that could affect other
 859  * things.
 860  */
 861 static void
 862 set_fencing_options(int level)
     /*  */
 863 {
 864     switch (level) {
 865         case 3:
 866             options.fence_connect = TRUE;
 867             fence_history = TRUE;
 868             show |= pcmk_section_fencing_all;
 869             break;
 870 
 871         case 2:
 872             options.fence_connect = TRUE;
 873             fence_history = TRUE;
 874             show |= pcmk_section_fencing_all;
 875             break;
 876 
 877         case 1:
 878             options.fence_connect = TRUE;
 879             fence_history = TRUE;
 880             show |= pcmk_section_fence_failed | pcmk_section_fence_pending;
 881             break;
 882 
 883         default:
 884             interactive_fence_level = 0;
 885             options.fence_connect = FALSE;
 886             fence_history = FALSE;
 887             show &= ~pcmk_section_fencing_all;
 888             break;
 889     }
 890 }
 891 
 892 /* Before trying to connect to fencer or cib check for state of
 893    pacemakerd - just no sense in trying till pacemakerd has
 894    taken care of starting all the sub-processes
 895 
 896    Only noteworthy thing to show here is when pacemakerd is
 897    waiting for startup-trigger from SBD.
 898  */
 899 static void
 900 pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
     /*  */
 901                     enum pcmk_ipc_event event_type, crm_exit_t status,
 902                     void *event_data, void *user_data)
 903 {
 904     pcmk_pacemakerd_api_reply_t *reply = event_data;
 905     enum pcmk_pacemakerd_state *state =
 906         (enum pcmk_pacemakerd_state *) user_data;
 907 
 908     /* we are just interested in the latest reply */
 909     *state = pcmk_pacemakerd_state_invalid;
 910 
 911     switch (event_type) {
 912         case pcmk_ipc_event_reply:
 913             break;
 914 
 915         default:
 916             return;
 917     }
 918 
 919     if (status != CRM_EX_OK) {
 920         out->err(out, "Bad reply from pacemakerd: %s",
 921                  crm_exit_str(status));
 922         return;
 923     }
 924 
 925     if (reply->reply_type != pcmk_pacemakerd_reply_ping) {
 926         out->err(out, "Unknown reply type %d from pacemakerd",
 927                  reply->reply_type);
 928     } else {
 929         if ((reply->data.ping.last_good != (time_t) 0) &&
 930             (reply->data.ping.status == pcmk_rc_ok)) {
 931             *state = reply->data.ping.state;
 932         }
 933     }
 934 }
 935 
 936 static int
 937 pacemakerd_status(void)
     /*  */
 938 {
 939     int rc = pcmk_rc_ok;
 940     pcmk_ipc_api_t *pacemakerd_api = NULL;
 941     enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;
 942 
 943     if (!use_cib_native) {
 944         /* we don't need fully functional pacemakerd otherwise */
 945         return rc;
 946     }
 947     if (cib != NULL &&
 948         (cib->state == cib_connected_query ||
 949          cib->state == cib_connected_command)) {
 950         /* As long as we have a cib-connection let's go with
 951          * that to fetch further cluster-status and avoid
 952          * unnecessary pings to pacemakerd.
 953          * If cluster is going down and fencer is down already
 954          * this will lead to a silently failing fencer reconnect.
 955          * On cluster startup we shouldn't see this situation
 956          * as first we do is wait for pacemakerd to report all
 957          * daemons running.
 958          */
 959         return rc;
 960     }
 961     rc = pcmk_new_ipc_api(&pacemakerd_api, pcmk_ipc_pacemakerd);
 962     if (pacemakerd_api == NULL) {
 963         out->err(out, "Could not connect to pacemakerd: %s",
 964                  pcmk_rc_str(rc));
 965         /* this is unrecoverable so return with rc we have */
 966         return rc;
 967     }
 968     pcmk_register_ipc_callback(pacemakerd_api, pacemakerd_event_cb, (void *) &state);
 969     rc = pcmk_connect_ipc(pacemakerd_api, pcmk_ipc_dispatch_poll);
 970     switch (rc) {
 971         case pcmk_rc_ok:
 972             rc = pcmk_pacemakerd_api_ping(pacemakerd_api, crm_system_name);
 973             if (rc == pcmk_rc_ok) {
 974                 rc = pcmk_poll_ipc(pacemakerd_api, options.reconnect_ms/2);
 975                 if (rc == pcmk_rc_ok) {
 976                     pcmk_dispatch_ipc(pacemakerd_api);
 977                     rc = ENOTCONN;
 978                     if ((output_format == mon_output_console) ||
 979                         (output_format == mon_output_plain)) {
 980                         switch (state) {
 981                             case pcmk_pacemakerd_state_running:
 982                                 rc = pcmk_rc_ok;
 983                                 break;
 984                             case pcmk_pacemakerd_state_starting_daemons:
 985                                 out->info(out,"Pacemaker daemons starting ...");
 986                                 break;
 987                             case pcmk_pacemakerd_state_wait_for_ping:
 988                                 out->info(out,"Waiting for startup-trigger from SBD ...");
 989                                 break;
 990                             case pcmk_pacemakerd_state_shutting_down:
 991                                 out->info(out,"Pacemaker daemons shutting down ...");
 992                                 /* try our luck maybe CIB is still accessible */
 993                                 rc = pcmk_rc_ok;
 994                                 break;
 995                             case pcmk_pacemakerd_state_shutdown_complete:
 996                                 /* assuming pacemakerd doesn't dispatch any pings after entering
 997                                 * that state unless it is waiting for SBD
 998                                 */
 999                                 out->info(out,"Pacemaker daemons shut down - reporting to SBD ...");
1000                                 break;
1001                             default:
1002                                 break;
1003                         }
1004                     } else {
1005                         switch (state) {
1006                             case pcmk_pacemakerd_state_running:
1007                                 rc = pcmk_rc_ok;
1008                                 break;
1009                             case pcmk_pacemakerd_state_shutting_down:
1010                                 /* try our luck maybe CIB is still accessible */
1011                                 rc = pcmk_rc_ok;
1012                                 break;
1013                             default:
1014                                 break;
1015                         }
1016                     }
1017                 }
1018             }
1019             break;
1020         case EREMOTEIO:
1021             rc = pcmk_rc_ok;
1022             on_remote_node = TRUE;
1023 #if CURSES_ENABLED
1024             /* just show this if refresh is gonna remove all traces */
1025             if (output_format == mon_output_console) {
1026                 out->info(out,
1027                     "Running on remote-node waiting to be connected by cluster ...");
1028             }
1029 #endif
1030             break;
1031         default:
1032             break;
1033     }
1034     pcmk_free_ipc_api(pacemakerd_api);
1035     /* returning with ENOTCONN triggers a retry */
1036     return (rc == pcmk_rc_ok)?rc:ENOTCONN;
1037 }
1038 
1039 #if CURSES_ENABLED
1040 static const char *
1041 get_option_desc(char c)
     /*  */
1042 {
1043     const char *desc = "No help available";
1044 
1045     for (GOptionEntry *entry = display_entries; entry != NULL; entry++) {
1046         if (entry->short_name == c) {
1047             desc = entry->description;
1048             break;
1049         }
1050     }
1051     return desc;
1052 }
1053 
1054 #define print_option_help(out, option, condition) \
1055     curses_formatted_printf(out, "%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
1056 
1057 /* This function is called from the main loop when there is something to be read
1058  * on stdin, like an interactive user's keystroke.  All it does is read the keystroke,
1059  * set flags (or show the page showing which keystrokes are valid), and redraw the
1060  * screen.  It does not do anything with connections to the CIB or fencing agent
1061  * agent what would happen in mon_refresh_display.
1062  */
1063 static gboolean
1064 detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer user_data)
     /*  */
1065 {
1066     int c;
1067     gboolean config_mode = FALSE;
1068 
1069     while (1) {
1070 
1071         /* Get user input */
1072         c = getchar();
1073 
1074         switch (c) {
1075             case 'm':
1076                 interactive_fence_level++;
1077                 if (interactive_fence_level > 3) {
1078                     interactive_fence_level = 0;
1079                 }
1080 
1081                 set_fencing_options(interactive_fence_level);
1082                 break;
1083             case 'c':
1084                 show ^= pcmk_section_tickets;
1085                 break;
1086             case 'f':
1087                 show ^= pcmk_section_failcounts;
1088                 break;
1089             case 'n':
1090                 show_opts ^= pcmk_show_rscs_by_node;
1091                 break;
1092             case 'o':
1093                 show ^= pcmk_section_operations;
1094                 if (!pcmk_is_set(show, pcmk_section_operations)) {
1095                     show_opts &= ~pcmk_show_timing;
1096                 }
1097                 break;
1098             case 'r':
1099                 show_opts ^= pcmk_show_inactive_rscs;
1100                 break;
1101             case 'R':
1102                 show_opts ^= pcmk_show_details;
1103 #ifdef PCMK__COMPAT_2_0
1104                 // Keep failed action output the same as 2.0.x
1105                 show_opts |= pcmk_show_failed_detail;
1106 #endif
1107                 break;
1108             case 't':
1109                 show_opts ^= pcmk_show_timing;
1110                 if (pcmk_is_set(show_opts, pcmk_show_timing)) {
1111                     show |= pcmk_section_operations;
1112                 }
1113                 break;
1114             case 'A':
1115                 show ^= pcmk_section_attributes;
1116                 break;
1117             case 'L':
1118                 show ^= pcmk_section_bans;
1119                 break;
1120             case 'D':
1121                 /* If any header is shown, clear them all, otherwise set them all */
1122                 if (pcmk_any_flags_set(show, pcmk_section_summary)) {
1123                     show &= ~pcmk_section_summary;
1124                 } else {
1125                     show |= pcmk_section_summary;
1126                 }
1127                 /* Regardless, we don't show options in console mode. */
1128                 show &= ~pcmk_section_options;
1129                 break;
1130             case 'b':
1131                 show_opts ^= pcmk_show_brief;
1132                 break;
1133             case 'j':
1134                 show_opts ^= pcmk_show_pending;
1135                 break;
1136             case '?':
1137                 config_mode = TRUE;
1138                 break;
1139             default:
1140                 /* All other keys just redraw the screen. */
1141                 goto refresh;
1142         }
1143 
1144         if (!config_mode)
1145             goto refresh;
1146 
1147         blank_screen();
1148 
1149         curses_formatted_printf(out, "%s", "Display option change mode\n");
1150         print_option_help(out, 'c', pcmk_is_set(show, pcmk_section_tickets));
1151         print_option_help(out, 'f', pcmk_is_set(show, pcmk_section_failcounts));
1152         print_option_help(out, 'n', pcmk_is_set(show_opts, pcmk_show_rscs_by_node));
1153         print_option_help(out, 'o', pcmk_is_set(show, pcmk_section_operations));
1154         print_option_help(out, 'r', pcmk_is_set(show_opts, pcmk_show_inactive_rscs));
1155         print_option_help(out, 't', pcmk_is_set(show_opts, pcmk_show_timing));
1156         print_option_help(out, 'A', pcmk_is_set(show, pcmk_section_attributes));
1157         print_option_help(out, 'L', pcmk_is_set(show, pcmk_section_bans));
1158         print_option_help(out, 'D', !pcmk_is_set(show, pcmk_section_summary));
1159         print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details));
1160         print_option_help(out, 'b', pcmk_is_set(show_opts, pcmk_show_brief));
1161         print_option_help(out, 'j', pcmk_is_set(show_opts, pcmk_show_pending));
1162         curses_formatted_printf(out, "%d m: \t%s\n", interactive_fence_level, get_option_desc('m'));
1163         curses_formatted_printf(out, "%s", "\nToggle fields via field letter, type any other key to return\n");
1164     }
1165 
1166 refresh:
1167     refresh_after_event(FALSE, TRUE);
1168 
1169     return TRUE;
1170 }
1171 #endif
1172 
1173 // Basically crm_signal_handler(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
1174 static void
1175 avoid_zombies(void)
     /*  */
1176 {
1177     struct sigaction sa;
1178 
1179     memset(&sa, 0, sizeof(struct sigaction));
1180     if (sigemptyset(&sa.sa_mask) < 0) {
1181         crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
1182         return;
1183     }
1184     sa.sa_handler = SIG_IGN;
1185     sa.sa_flags = SA_RESTART|SA_NOCLDWAIT;
1186     if (sigaction(SIGCHLD, &sa, NULL) < 0) {
1187         crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
1188     }
1189 }
1190 
1191 static GOptionContext *
1192 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     /*  */
1193     GOptionContext *context = NULL;
1194 
1195     GOptionEntry extra_prog_entries[] = {
1196         { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
1197           "Be less descriptive in output.",
1198           NULL },
1199 
1200         { NULL }
1201     };
1202 
1203     const char *description = "Notes:\n\n"
1204                               "If this program is called as crm_mon.cgi, --output-as=html --html-cgi will\n"
1205                               "automatically be added to the command line arguments.\n\n"
1206                               "Time Specification:\n\n"
1207                               "The TIMESPEC in any command line option can be specified in many different\n"
1208                               "formats.  It can be just an integer number of seconds, a number plus units\n"
1209                               "(ms/msec/us/usec/s/sec/m/min/h/hr), or an ISO 8601 period specification.\n\n"
1210                               "Output Control:\n\n"
1211                               "By default, a certain list of sections are written to the output destination.\n"
1212                               "The default varies based on the output format - XML includes everything, while\n"
1213                               "other output formats will display less.  This list can be modified with the\n"
1214                               "--include and --exclude command line options.  Each option may be given multiple\n"
1215                               "times on the command line, and each can give a comma-separated list of sections.\n"
1216                               "The options are applied to the default set, from left to right as seen on the\n"
1217                               "command line.  For a list of valid sections, pass --include=list or --exclude=list.\n\n"
1218                               "Interactive Use:\n\n"
1219                               "When run interactively, crm_mon can be told to hide and display various sections\n"
1220                               "of output.  To see a help screen explaining the options, hit '?'.  Any key stroke\n"
1221                               "aside from those listed will cause the screen to refresh.\n\n"
1222                               "Examples:\n\n"
1223                               "Display the cluster status on the console with updates as they occur:\n\n"
1224                               "\tcrm_mon\n\n"
1225                               "Display the cluster status on the console just once then exit:\n\n"
1226                               "\tcrm_mon -1\n\n"
1227                               "Display your cluster status, group resources by node, and include inactive resources in the list:\n\n"
1228                               "\tcrm_mon --group-by-node --inactive\n\n"
1229                               "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:\n\n"
1230                               "\tcrm_mon --daemonize --output-as html --output-to /path/to/docroot/filename.html\n\n"
1231                               "Start crm_mon and export the current cluster status as XML to stdout, then exit:\n\n"
1232                               "\tcrm_mon --output-as xml\n\n";
1233 
1234 #if CURSES_ENABLED
1235     context = pcmk__build_arg_context(args, "console (default), html, text, xml", group, NULL);
1236 #else
1237     context = pcmk__build_arg_context(args, "text (default), html, xml", group, NULL);
1238 #endif
1239     pcmk__add_main_args(context, extra_prog_entries);
1240     g_option_context_set_description(context, description);
1241 
1242     pcmk__add_arg_group(context, "display", "Display Options:",
1243                         "Show display options", display_entries);
1244     pcmk__add_arg_group(context, "additional", "Additional Options:",
1245                         "Show additional options", addl_entries);
1246     pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
1247                         "Show deprecated options", deprecated_entries);
1248 
1249     return context;
1250 }
1251 
1252 /* If certain format options were specified, we want to set some extra
1253  * options.  We can just process these like they were given on the
1254  * command line.
1255  */
1256 static void
1257 add_output_args(void) {
     /*  */
1258     GError *err = NULL;
1259 
1260     if (output_format == mon_output_plain) {
1261         if (!pcmk__force_args(context, &err, "%s --text-fancy", g_get_prgname())) {
1262             g_propagate_error(&error, err);
1263             clean_up(CRM_EX_USAGE);
1264         }
1265     } else if (output_format == mon_output_cgi) {
1266         if (!pcmk__force_args(context, &err, "%s --html-cgi", g_get_prgname())) {
1267             g_propagate_error(&error, err);
1268             clean_up(CRM_EX_USAGE);
1269         }
1270     } else if (output_format == mon_output_xml) {
1271         if (!pcmk__force_args(context, &err, "%s --xml-simple-list --xml-substitute", g_get_prgname())) {
1272             g_propagate_error(&error, err);
1273             clean_up(CRM_EX_USAGE);
1274         }
1275     } else if (output_format == mon_output_legacy_xml) {
1276         output_format = mon_output_xml;
1277         if (!pcmk__force_args(context, &err, "%s --xml-legacy --xml-substitute", g_get_prgname())) {
1278             g_propagate_error(&error, err);
1279             clean_up(CRM_EX_USAGE);
1280         }
1281     }
1282 }
1283 
1284 /* Which output format to use could come from two places:  The --as-xml
1285  * style arguments we gave in deprecated_entries above, or the formatted output
1286  * arguments added by pcmk__register_formats.  If the latter were used,
1287  * output_format will be mon_output_unset.
1288  *
1289  * Call the callbacks as if those older style arguments were provided so
1290  * the various things they do get done.
1291  */
1292 static void
1293 reconcile_output_format(pcmk__common_args_t *args) {
     /*  */
1294     gboolean retval = TRUE;
1295     GError *err = NULL;
1296 
1297     if (output_format != mon_output_unset) {
1298         return;
1299     }
1300 
1301     if (pcmk__str_eq(args->output_ty, "html", pcmk__str_casei)) {
1302         char *dest = NULL;
1303 
1304         if (args->output_dest != NULL) {
1305             dest = strdup(args->output_dest);
1306         }
1307 
1308         retval = as_html_cb("h", dest, NULL, &err);
1309         free(dest);
1310     } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_casei)) {
1311         retval = no_curses_cb("N", NULL, NULL, &err);
1312     } else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_casei)) {
1313         if (args->output_ty != NULL) {
1314             free(args->output_ty);
1315         }
1316 
1317         args->output_ty = strdup("xml");
1318         output_format = mon_output_xml;
1319     } else if (options.one_shot) {
1320         if (args->output_ty != NULL) {
1321             free(args->output_ty);
1322         }
1323 
1324         args->output_ty = strdup("text");
1325         output_format = mon_output_plain;
1326     } else {
1327         /* Neither old nor new arguments were given, so set the default. */
1328         if (args->output_ty != NULL) {
1329             free(args->output_ty);
1330         }
1331 
1332         args->output_ty = strdup("console");
1333         output_format = mon_output_console;
1334     }
1335 
1336     if (!retval) {
1337         g_propagate_error(&error, err);
1338         clean_up(CRM_EX_USAGE);
1339     }
1340 }
1341 
1342 static void
1343 handle_connection_failures(int rc)
     /*  */
1344 {
1345     if (rc == pcmk_rc_ok) {
1346         return;
1347     }
1348 
1349     if (output_format == mon_output_monitor) {
1350         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "CLUSTER CRIT: Connection to cluster failed: %s",
1351                     pcmk_rc_str(rc));
1352         rc = MON_STATUS_CRIT;
1353     } else if (rc == ENOTCONN) {
1354         if (on_remote_node) {
1355             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: remote-node not connected to cluster");
1356         } else {
1357             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");
1358         }
1359         rc = pcmk_rc2exitc(rc);
1360     } else {
1361         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Connection to cluster failed: %s", pcmk_rc_str(rc));
1362         rc = pcmk_rc2exitc(rc);
1363     }
1364 
1365     clean_up(rc);
1366 }
1367 
1368 static void
1369 one_shot(void)
     /*  */
1370 {
1371     int rc;
1372 
1373     rc = pacemakerd_status();
1374 
1375     if (rc == pcmk_rc_ok) {
1376         fencing_connect();
1377         rc = cib_connect(FALSE);
1378     }
1379 
1380     if (rc == pcmk_rc_ok) {
1381         mon_refresh_display(NULL);
1382     } else {
1383         handle_connection_failures(rc);
1384     }
1385 
1386     clean_up(CRM_EX_OK);
1387 }
1388 
1389 int
1390 main(int argc, char **argv)
     /*  */
1391 {
1392     int rc = pcmk_ok;
1393     GOptionGroup *output_group = NULL;
1394 
1395     args = pcmk__new_common_args(SUMMARY);
1396     context = build_arg_context(args, &output_group);
1397     pcmk__register_formats(output_group, formats);
1398 
1399     options.pid_file = strdup("/tmp/ClusterMon.pid");
1400     pcmk__cli_init_logging("crm_mon", 0);
1401 
1402     // Avoid needing to wait for subprocesses forked for -E/--external-agent
1403     avoid_zombies();
1404 
1405     if (pcmk__ends_with_ext(argv[0], ".cgi")) {
1406         output_format = mon_output_cgi;
1407         options.one_shot = TRUE;
1408     }
1409 
1410     processed_args = pcmk__cmdline_preproc(argv, "ehimpxEILU");
1411 
1412     fence_history_cb("--fence-history", "1", NULL, NULL);
1413 
1414     /* Set an HTML title regardless of what format we will eventually use.  This can't
1415      * be done in add_output_args.  That function is called after command line
1416      * arguments are processed in the next block, which means it'll override whatever
1417      * title the user provides.  Doing this here means the user can give their own
1418      * title on the command line.
1419      */
1420     if (!pcmk__force_args(context, &error, "%s --html-title \"Cluster Status\"",
1421                           g_get_prgname())) {
1422         return clean_up(CRM_EX_USAGE);
1423     }
1424 
1425     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
1426         return clean_up(CRM_EX_USAGE);
1427     }
1428 
1429     for (int i = 0; i < args->verbosity; i++) {
1430         crm_bump_log_level(argc, argv);
1431     }
1432 
1433     if (!args->version) {
1434         if (args->quiet) {
1435             include_exclude_cb("--exclude", "times", NULL, NULL);
1436         }
1437 
1438         if (options.watch_fencing) {
1439             fence_history_cb("--fence-history", "0", NULL, NULL);
1440             options.fence_connect = TRUE;
1441         }
1442 
1443         /* create the cib-object early to be able to do further
1444          * decisions based on the cib-source
1445          */
1446         cib = cib_new();
1447 
1448         if (cib == NULL) {
1449             rc = -EINVAL;
1450         } else {
1451             switch (cib->variant) {
1452 
1453                 case cib_native:
1454                     /* cib & fencing - everything available */
1455                     use_cib_native = TRUE;
1456                     break;
1457 
1458                 case cib_file:
1459                     /* Don't try to connect to fencing as we
1460                      * either don't have a running cluster or
1461                      * the fencing-information would possibly
1462                      * not match the cib data from a file.
1463                      * As we don't expect cib-updates coming
1464                      * in enforce one-shot. */
1465                     fence_history_cb("--fence-history", "0", NULL, NULL);
1466                     options.one_shot = TRUE;
1467                     break;
1468 
1469                 case cib_remote:
1470                     /* updates coming in but no fencing */
1471                     fence_history_cb("--fence-history", "0", NULL, NULL);
1472                     break;
1473 
1474                 case cib_undefined:
1475                 case cib_database:
1476                 default:
1477                     /* something is odd */
1478                     rc = -EINVAL;
1479                     break;
1480             }
1481         }
1482 
1483         if (options.one_shot) {
1484             if (output_format == mon_output_console) {
1485                 output_format = mon_output_plain;
1486             }
1487 
1488         } else if (options.daemonize) {
1489             if ((output_format == mon_output_console) || (args->output_dest == NULL)) {
1490                 output_format = mon_output_none;
1491             }
1492             crm_enable_stderr(FALSE);
1493 
1494             if (pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches | pcmk__str_casei) && !options.external_agent) {
1495                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "--daemonize requires at least one of --output-to and --external-agent");
1496                 return clean_up(CRM_EX_USAGE);
1497             }
1498 
1499             if (cib) {
1500                 /* to be on the safe side don't have cib-object around
1501                  * when we are forking
1502                  */
1503                 cib_delete(cib);
1504                 cib = NULL;
1505                 pcmk__daemonize(crm_system_name, options.pid_file);
1506                 cib = cib_new();
1507                 if (cib == NULL) {
1508                     rc = -EINVAL;
1509                 }
1510                 /* otherwise assume we've got the same cib-object we've just destroyed
1511                  * in our parent
1512                  */
1513             }
1514 
1515 
1516         } else if (output_format == mon_output_console) {
1517 #if CURSES_ENABLED
1518             crm_enable_stderr(FALSE);
1519 #else
1520             options.one_shot = TRUE;
1521             output_format = mon_output_plain;
1522             printf("Defaulting to one-shot mode\n");
1523             printf("You need to have curses available at compile time to enable console mode\n");
1524 #endif
1525         }
1526     }
1527 
1528     if (rc != pcmk_ok) {
1529         // Shouldn't really be possible
1530         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Invalid CIB source");
1531         return clean_up(CRM_EX_ERROR);
1532     }
1533 
1534     reconcile_output_format(args);
1535     add_output_args();
1536 
1537     if (args->version && output_format == mon_output_console) {
1538         /* Use the text output format here if we are in curses mode but were given
1539          * --version.  Displaying version information uses printf, and then we
1540          *  immediately exit.  We don't want to initialize curses for that.
1541          */
1542         rc = pcmk__output_new(&out, "text", args->output_dest, argv);
1543     } else {
1544         rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
1545     }
1546 
1547     if (rc != pcmk_rc_ok) {
1548         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error creating output format %s: %s",
1549                     args->output_ty, pcmk_rc_str(rc));
1550         return clean_up(CRM_EX_ERROR);
1551     }
1552 
1553     /* output_format MUST NOT BE CHANGED AFTER THIS POINT. */
1554 
1555     show = default_includes(output_format);
1556 
1557     /* Apply --include/--exclude flags we used internally.  There's no error reporting
1558      * here because this would be a programming error.
1559      */
1560     apply_include_exclude(options.includes_excludes, &error);
1561 
1562     /* And now apply any --include/--exclude flags the user gave on the command line.
1563      * These are done in a separate pass from the internal ones because we want to
1564      * make sure whatever the user specifies overrides whatever we do.
1565      */
1566     if (!apply_include_exclude(options.user_includes_excludes, &error)) {
1567         return clean_up(CRM_EX_USAGE);
1568     }
1569 
1570     /* Sync up the initial value of interactive_fence_level with whatever was set with
1571      * --include/--exclude= options.
1572      */
1573     if (pcmk_all_flags_set(show, pcmk_section_fencing_all)) {
1574         interactive_fence_level = 3;
1575     } else if (pcmk_is_set(show, pcmk_section_fence_worked)) {
1576         interactive_fence_level = 2;
1577     } else if (pcmk_any_flags_set(show, pcmk_section_fence_failed | pcmk_section_fence_pending)) {
1578         interactive_fence_level = 1;
1579     } else {
1580         interactive_fence_level = 0;
1581     }
1582 
1583     pcmk__register_lib_messages(out);
1584     crm_mon_register_messages(out);
1585     pe__register_messages(out);
1586     stonith__register_messages(out);
1587 
1588     if (args->version) {
1589         out->version(out, false);
1590         return clean_up(CRM_EX_OK);
1591     }
1592 
1593     /* Extra sanity checks when in CGI mode */
1594     if (output_format == mon_output_cgi) {
1595         if (cib->variant == cib_file) {
1596             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode used with CIB file");
1597             return clean_up(CRM_EX_USAGE);
1598         } else if (options.external_agent != NULL) {
1599             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with --external-agent");
1600             return clean_up(CRM_EX_USAGE);
1601         } else if (options.daemonize == TRUE) {
1602             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with -d");
1603             return clean_up(CRM_EX_USAGE);
1604         }
1605     }
1606 
1607     if (output_format == mon_output_xml || output_format == mon_output_legacy_xml) {
1608         show_opts |= pcmk_show_inactive_rscs | pcmk_show_timing;
1609 
1610         if (!options.daemonize) {
1611             options.one_shot = TRUE;
1612         }
1613     }
1614 
1615     if ((output_format == mon_output_html || output_format == mon_output_cgi) &&
1616         out->dest != stdout) {
1617         pcmk__html_add_header("meta", "http-equiv", "refresh", "content",
1618                               pcmk__itoa(options.reconnect_ms / 1000), NULL);
1619     }
1620 
1621 #ifdef PCMK__COMPAT_2_0
1622     // Keep failed action output the same as 2.0.x
1623     show_opts |= pcmk_show_failed_detail;
1624 #endif
1625 
1626     crm_info("Starting %s", crm_system_name);
1627 
1628     cib__set_output(cib, out);
1629 
1630     if (options.one_shot) {
1631         one_shot();
1632     }
1633 
1634     do {
1635         out->info(out,"Waiting until cluster is available on this node ...");
1636 
1637         rc = pacemakerd_status();
1638         if (rc == pcmk_rc_ok) {
1639             fencing_connect();
1640             rc = cib_connect(TRUE);
1641         }
1642 
1643         if (rc != pcmk_rc_ok) {
1644             pcmk__sleep_ms(options.reconnect_ms);
1645 #if CURSES_ENABLED
1646             if (output_format == mon_output_console) {
1647                 clear();
1648                 refresh();
1649             }
1650 #endif
1651         } else if (output_format == mon_output_html && out->dest != stdout) {
1652             printf("Writing html to %s ...\n", args->output_dest);
1653         }
1654 
1655     } while (rc == ENOTCONN);
1656 
1657     handle_connection_failures(rc);
1658     set_fencing_options(interactive_fence_level);
1659     mon_refresh_display(NULL);
1660 
1661     mainloop = g_main_loop_new(NULL, FALSE);
1662 
1663     mainloop_add_signal(SIGTERM, mon_shutdown);
1664     mainloop_add_signal(SIGINT, mon_shutdown);
1665 #if CURSES_ENABLED
1666     if (output_format == mon_output_console) {
1667         ncurses_winch_handler = crm_signal_handler(SIGWINCH, mon_winresize);
1668         if (ncurses_winch_handler == SIG_DFL ||
1669             ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
1670             ncurses_winch_handler = NULL;
1671 
1672         io_channel = g_io_channel_unix_new(STDIN_FILENO);
1673         g_io_add_watch(io_channel, G_IO_IN, detect_user_input, NULL);
1674     }
1675 #endif
1676 
1677     /* When refresh_trigger->trigger is set to TRUE, call mon_refresh_display.  In
1678      * this file, that is anywhere mainloop_set_trigger is called.
1679      */
1680     refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
1681 
1682     g_main_loop_run(mainloop);
1683     g_main_loop_unref(mainloop);
1684 
1685     if (io_channel != NULL) {
1686         g_io_channel_shutdown(io_channel, TRUE, NULL);
1687     }
1688 
1689     crm_info("Exiting %s", crm_system_name);
1690 
1691     return clean_up(CRM_EX_OK);
1692 }
1693 
1694 /*!
1695  * \internal
1696  * \brief Print one-line status suitable for use with monitoring software
1697  *
1698  * \param[in] data_set  Working set of CIB state
1699  *
1700  * \note This function's output (and the return code when the program exits)
1701  *       should conform to https://www.monitoring-plugins.org/doc/guidelines.html
1702  */
1703 static void
1704 print_simple_status(pcmk__output_t *out, pe_working_set_t * data_set)
     /*  */
1705 {
1706     GList *gIter = NULL;
1707     int nodes_online = 0;
1708     int nodes_standby = 0;
1709     int nodes_maintenance = 0;
1710     char *offline_nodes = NULL;
1711     size_t offline_nodes_len = 0;
1712     gboolean no_dc = FALSE;
1713     gboolean offline = FALSE;
1714 
1715     if (data_set->dc_node == NULL) {
1716         has_warnings = TRUE;
1717         no_dc = TRUE;
1718     }
1719 
1720     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1721         pe_node_t *node = (pe_node_t *) gIter->data;
1722 
1723         if (node->details->standby && node->details->online) {
1724             nodes_standby++;
1725         } else if (node->details->maintenance && node->details->online) {
1726             nodes_maintenance++;
1727         } else if (node->details->online) {
1728             nodes_online++;
1729         } else {
1730             char *s = crm_strdup_printf("offline node: %s", node->details->uname);
1731             /* coverity[leaked_storage] False positive */
1732             pcmk__add_word(&offline_nodes, &offline_nodes_len, s);
1733             free(s);
1734             has_warnings = TRUE;
1735             offline = TRUE;
1736         }
1737     }
1738 
1739     if (has_warnings) {
1740         out->info(out, "CLUSTER WARN: %s%s%s",
1741                   no_dc ? "No DC" : "",
1742                   no_dc && offline ? ", " : "",
1743                   (offline? offline_nodes : ""));
1744         free(offline_nodes);
1745     } else {
1746         char *nodes_standby_s = NULL;
1747         char *nodes_maint_s = NULL;
1748 
1749         if (nodes_standby > 0) {
1750             nodes_standby_s = crm_strdup_printf(", %d standby node%s", nodes_standby,
1751                                                 pcmk__plural_s(nodes_standby));
1752         }
1753 
1754         if (nodes_maintenance > 0) {
1755             nodes_maint_s = crm_strdup_printf(", %d maintenance node%s",
1756                                               nodes_maintenance,
1757                                               pcmk__plural_s(nodes_maintenance));
1758         }
1759 
1760         out->info(out, "CLUSTER OK: %d node%s online%s%s, "
1761                        "%d resource instance%s configured",
1762                   nodes_online, pcmk__plural_s(nodes_online),
1763                   nodes_standby_s != NULL ? nodes_standby_s : "",
1764                   nodes_maint_s != NULL ? nodes_maint_s : "",
1765                   data_set->ninstances, pcmk__plural_s(data_set->ninstances));
1766 
1767         free(nodes_standby_s);
1768         free(nodes_maint_s);
1769     }
1770     /* coverity[leaked_storage] False positive */
1771 }
1772 
1773 static int
1774 send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
     /*  */
1775                  int status, const char *desc)
1776 {
1777     pid_t pid;
1778 
1779     /*setenv needs chars, these are ints */
1780     char *rc_s = pcmk__itoa(rc);
1781     char *status_s = pcmk__itoa(status);
1782     char *target_rc_s = pcmk__itoa(target_rc);
1783 
1784     crm_debug("Sending external notification to '%s' via '%s'", options.external_recipient, options.external_agent);
1785 
1786     if(rsc) {
1787         setenv("CRM_notify_rsc", rsc, 1);
1788     }
1789     if (options.external_recipient) {
1790         setenv("CRM_notify_recipient", options.external_recipient, 1);
1791     }
1792     setenv("CRM_notify_node", node, 1);
1793     setenv("CRM_notify_task", task, 1);
1794     setenv("CRM_notify_desc", desc, 1);
1795     setenv("CRM_notify_rc", rc_s, 1);
1796     setenv("CRM_notify_target_rc", target_rc_s, 1);
1797     setenv("CRM_notify_status", status_s, 1);
1798 
1799     pid = fork();
1800     if (pid == -1) {
1801         crm_perror(LOG_ERR, "notification fork() failed.");
1802     }
1803     if (pid == 0) {
1804         /* crm_debug("notification: I am the child. Executing the nofitication program."); */
1805         execl(options.external_agent, options.external_agent, NULL);
1806         exit(CRM_EX_ERROR);
1807     }
1808 
1809     crm_trace("Finished running custom notification program '%s'.", options.external_agent);
1810     free(target_rc_s);
1811     free(status_s);
1812     free(rc_s);
1813     return 0;
1814 }
1815 
1816 static void
1817 handle_rsc_op(xmlNode * xml, const char *node_id)
     /*  */
1818 {
1819     int rc = -1;
1820     int status = -1;
1821     int target_rc = -1;
1822     gboolean notify = TRUE;
1823 
1824     char *rsc = NULL;
1825     char *task = NULL;
1826     const char *desc = NULL;
1827     const char *magic = NULL;
1828     const char *id = NULL;
1829     const char *node = NULL;
1830 
1831     xmlNode *n = xml;
1832     xmlNode * rsc_op = xml;
1833 
1834     if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
1835         xmlNode *cIter;
1836 
1837         for(cIter = xml->children; cIter; cIter = cIter->next) {
1838             handle_rsc_op(cIter, node_id);
1839         }
1840 
1841         return;
1842     }
1843 
1844     id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
1845     if (id == NULL) {
1846         /* Compatibility with <= 1.1.5 */
1847         id = ID(rsc_op);
1848     }
1849 
1850     magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
1851     if (magic == NULL) {
1852         /* non-change */
1853         return;
1854     }
1855 
1856     if (!decode_transition_magic(magic, NULL, NULL, NULL, &status, &rc,
1857                                  &target_rc)) {
1858         crm_err("Invalid event %s detected for %s", magic, id);
1859         return;
1860     }
1861 
1862     if (parse_op_key(id, &rsc, &task, NULL) == FALSE) {
1863         crm_err("Invalid event detected for %s", id);
1864         goto bail;
1865     }
1866 
1867     node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
1868 
1869     while (n != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(n), pcmk__str_casei)) {
1870         n = n->parent;
1871     }
1872 
1873     if(node == NULL && n) {
1874         node = crm_element_value(n, XML_ATTR_UNAME);
1875     }
1876 
1877     if (node == NULL && n) {
1878         node = ID(n);
1879     }
1880 
1881     if (node == NULL) {
1882         node = node_id;
1883     }
1884 
1885     if (node == NULL) {
1886         crm_err("No node detected for event %s (%s)", magic, id);
1887         goto bail;
1888     }
1889 
1890     /* look up where we expected it to be? */
1891     desc = pcmk_strerror(pcmk_ok);
1892     if ((status == PCMK_EXEC_DONE) && (target_rc == rc)) {
1893         crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
1894         if (rc == PCMK_OCF_NOT_RUNNING) {
1895             notify = FALSE;
1896         }
1897 
1898     } else if (status == PCMK_EXEC_DONE) {
1899         desc = services_ocf_exitcode_str(rc);
1900         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1901 
1902     } else {
1903         desc = pcmk_exec_status_str(status);
1904         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1905     }
1906 
1907     if (notify && options.external_agent) {
1908         send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
1909     }
1910   bail:
1911     free(rsc);
1912     free(task);
1913 }
1914 
1915 /* This function is just a wrapper around mainloop_set_trigger so that it can be
1916  * called from a mainloop directly.  It's simply another way of ensuring the screen
1917  * gets redrawn.
1918  */
1919 static gboolean
1920 mon_trigger_refresh(gpointer user_data)
     /*  */
1921 {
1922     mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
1923     return FALSE;
1924 }
1925 
1926 static void
1927 crm_diff_update_v2(const char *event, xmlNode * msg)
     /*  */
1928 {
1929     xmlNode *change = NULL;
1930     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
1931 
1932     for (change = pcmk__xml_first_child(diff); change != NULL;
1933          change = pcmk__xml_next(change)) {
1934         const char *name = NULL;
1935         const char *op = crm_element_value(change, XML_DIFF_OP);
1936         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
1937         xmlNode *match = NULL;
1938         const char *node = NULL;
1939 
1940         if(op == NULL) {
1941             continue;
1942 
1943         } else if(strcmp(op, "create") == 0) {
1944             match = change->children;
1945 
1946         } else if(strcmp(op, "move") == 0) {
1947             continue;
1948 
1949         } else if(strcmp(op, "delete") == 0) {
1950             continue;
1951 
1952         } else if(strcmp(op, "modify") == 0) {
1953             match = first_named_child(change, XML_DIFF_RESULT);
1954             if(match) {
1955                 match = match->children;
1956             }
1957         }
1958 
1959         if(match) {
1960             name = (const char *)match->name;
1961         }
1962 
1963         crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
1964         if(xpath == NULL) {
1965             /* Version field, ignore */
1966 
1967         } else if(name == NULL) {
1968             crm_debug("No result for %s operation to %s", op, xpath);
1969             CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
1970 
1971         } else if(strcmp(name, XML_TAG_CIB) == 0) {
1972             xmlNode *state = NULL;
1973             xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
1974 
1975             for (state = pcmk__xe_first_child(status); state != NULL;
1976                  state = pcmk__xe_next(state)) {
1977 
1978                 node = crm_element_value(state, XML_ATTR_UNAME);
1979                 if (node == NULL) {
1980                     node = ID(state);
1981                 }
1982                 handle_rsc_op(state, node);
1983             }
1984 
1985         } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
1986             xmlNode *state = NULL;
1987 
1988             for (state = pcmk__xe_first_child(match); state != NULL;
1989                  state = pcmk__xe_next(state)) {
1990 
1991                 node = crm_element_value(state, XML_ATTR_UNAME);
1992                 if (node == NULL) {
1993                     node = ID(state);
1994                 }
1995                 handle_rsc_op(state, node);
1996             }
1997 
1998         } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
1999             node = crm_element_value(match, XML_ATTR_UNAME);
2000             if (node == NULL) {
2001                 node = ID(match);
2002             }
2003             handle_rsc_op(match, node);
2004 
2005         } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
2006             node = ID(match);
2007             handle_rsc_op(match, node);
2008 
2009         } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
2010             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2011 
2012             handle_rsc_op(match, local_node);
2013             free(local_node);
2014 
2015         } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
2016             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2017 
2018             handle_rsc_op(match, local_node);
2019             free(local_node);
2020 
2021         } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
2022             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2023 
2024             handle_rsc_op(match, local_node);
2025             free(local_node);
2026 
2027         } else {
2028             crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
2029         }
2030     }
2031 }
2032 
2033 static void
2034 crm_diff_update_v1(const char *event, xmlNode * msg)
     /*  */
2035 {
2036     /* Process operation updates */
2037     xmlXPathObject *xpathObj = xpath_search(msg,
2038                                             "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
2039                                             "//" XML_LRM_TAG_RSC_OP);
2040     int lpc = 0, max = numXpathResults(xpathObj);
2041 
2042     for (lpc = 0; lpc < max; lpc++) {
2043         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
2044 
2045         handle_rsc_op(rsc_op, NULL);
2046     }
2047     freeXpathObject(xpathObj);
2048 }
2049 
2050 static void
2051 crm_diff_update(const char *event, xmlNode * msg)
     /*  */
2052 {
2053     int rc = -1;
2054     static bool stale = FALSE;
2055     gboolean cib_updated = FALSE;
2056     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
2057 
2058     out->progress(out, false);
2059 
2060     if (current_cib != NULL) {
2061         rc = xml_apply_patchset(current_cib, diff, TRUE);
2062 
2063         switch (rc) {
2064             case -pcmk_err_diff_resync:
2065             case -pcmk_err_diff_failed:
2066                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
2067                 free_xml(current_cib); current_cib = NULL;
2068                 break;
2069             case pcmk_ok:
2070                 cib_updated = TRUE;
2071                 break;
2072             default:
2073                 crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
2074                 free_xml(current_cib); current_cib = NULL;
2075         }
2076     }
2077 
2078     if (current_cib == NULL) {
2079         crm_trace("Re-requesting the full cib");
2080         cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
2081     }
2082 
2083     if (options.external_agent) {
2084         int format = 0;
2085         crm_element_value_int(diff, "format", &format);
2086         switch(format) {
2087             case 1:
2088                 crm_diff_update_v1(event, msg);
2089                 break;
2090             case 2:
2091                 crm_diff_update_v2(event, msg);
2092                 break;
2093             default:
2094                 crm_err("Unknown patch format: %d", format);
2095         }
2096     }
2097 
2098     if (current_cib == NULL) {
2099         if(!stale) {
2100             out->info(out, "--- Stale data ---");
2101         }
2102         stale = TRUE;
2103         return;
2104     }
2105 
2106     stale = FALSE;
2107     refresh_after_event(cib_updated, FALSE);
2108 }
2109 
2110 static int
2111 get_fencing_history(stonith_history_t **stonith_history)
     /*  */
2112 {
2113     int rc = 0;
2114 
2115     while (fence_history) {
2116         if (st != NULL) {
2117             rc = st->cmds->history(st, st_opt_sync_call, NULL, stonith_history, 120);
2118 
2119             if (rc == 0) {
2120                 *stonith_history = stonith__sort_history(*stonith_history);
2121                 if (!pcmk_all_flags_set(show, pcmk_section_fencing_all)
2122                     && (output_format != mon_output_xml)) {
2123 
2124                     *stonith_history = pcmk__reduce_fence_history(*stonith_history);
2125                 }
2126                 break; /* all other cases are errors */
2127             }
2128         } else {
2129             rc = ENOTCONN;
2130             break;
2131         }
2132     }
2133 
2134     return rc;
2135 }
2136 
2137 static int
2138 mon_refresh_display(gpointer user_data)
     /*  */
2139 {
2140     xmlNode *cib_copy = copy_xml(current_cib);
2141     stonith_history_t *stonith_history = NULL;
2142     int history_rc = 0;
2143     GList *unames = NULL;
2144     GList *resources = NULL;
2145 
2146     last_refresh = time(NULL);
2147 
2148     if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
2149         clean_up_cib_connection();
2150         out->err(out, "Upgrade failed: %s", pcmk_strerror(-pcmk_err_schema_validation));
2151         clean_up(CRM_EX_CONFIG);
2152         return 0;
2153     }
2154 
2155     /* get the stonith-history if there is evidence we need it */
2156     history_rc = get_fencing_history(&stonith_history);
2157 
2158     if (mon_data_set == NULL) {
2159         mon_data_set = pe_new_working_set();
2160         CRM_ASSERT(mon_data_set != NULL);
2161     }
2162     pe__set_working_set_flags(mon_data_set, pe_flag_no_compat);
2163 
2164     mon_data_set->input = cib_copy;
2165     mon_data_set->priv = out;
2166     cluster_status(mon_data_set);
2167 
2168     /* Unpack constraints if any section will need them
2169      * (tickets may be referenced in constraints but not granted yet,
2170      * and bans need negative location constraints) */
2171     if (pcmk_is_set(show, pcmk_section_bans) || pcmk_is_set(show, pcmk_section_tickets)) {
2172         pcmk__unpack_constraints(mon_data_set);
2173     }
2174 
2175     if (options.daemonize) {
2176         out->reset(out);
2177     }
2178 
2179     unames = pe__build_node_name_list(mon_data_set, options.only_node);
2180     resources = pe__build_rsc_list(mon_data_set, options.only_rsc);
2181 
2182     /* Always print DC if NULL. */
2183     if (mon_data_set->dc_node == NULL) {
2184         show |= pcmk_section_dc;
2185     }
2186 
2187     switch (output_format) {
2188         case mon_output_html:
2189         case mon_output_cgi:
2190             if (out->message(out, "cluster-status", mon_data_set, crm_errno2exit(history_rc),
2191                              stonith_history, fence_history, show, show_opts,
2192                              options.neg_location_prefix, unames, resources) != 0) {
2193                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_CANTCREAT, "Critical: Unable to output html file");
2194                 clean_up(CRM_EX_CANTCREAT);
2195                 return 0;
2196             }
2197             break;
2198 
2199         case mon_output_monitor:
2200             print_simple_status(out, mon_data_set);
2201             if (has_warnings) {
2202                 clean_up(MON_STATUS_WARN);
2203                 return FALSE;
2204             }
2205             break;
2206 
2207         case mon_output_unset:
2208         case mon_output_none:
2209             break;
2210 
2211         default:
2212             out->message(out, "cluster-status", mon_data_set, crm_errno2exit(history_rc),
2213                          stonith_history, fence_history, show, show_opts,
2214                          options.neg_location_prefix, unames, resources);
2215             break;
2216     }
2217 
2218     if (options.daemonize) {
2219         out->finish(out, CRM_EX_OK, true, NULL);
2220     }
2221 
2222     g_list_free_full(unames, free);
2223     g_list_free_full(resources, free);
2224 
2225     stonith_history_free(stonith_history);
2226     stonith_history = NULL;
2227     pe_reset_working_set(mon_data_set);
2228     return 1;
2229 }
2230 
2231 /* This function is called for fencing events (see fencing_connect for which ones) when
2232  * --watch-fencing is used on the command line.
2233  */
2234 static void
2235 mon_st_callback_event(stonith_t * st, stonith_event_t * e)
     /*  */
2236 {
2237     if (st->state == stonith_disconnected) {
2238         /* disconnect cib as well and have everything reconnect */
2239         mon_cib_connection_destroy(NULL);
2240     } else if (options.external_agent) {
2241         char *desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
2242                                     e->operation, e->origin, e->target, pcmk_strerror(e->result),
2243                                     e->id);
2244         send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
2245         free(desc);
2246     }
2247 }
2248 
2249 /* Cause the screen to be redrawn (via mainloop_set_trigger) when various conditions are met:
2250  *
2251  * - If the last update occurred more than reconnect_ms ago (defaults to 5s, but
2252  *   can be changed via the -i command line option), or
2253  * - After every 10 CIB updates, or
2254  * - If it's been 2s since the last update
2255  *
2256  * This function sounds like it would be more broadly useful, but it is only called when a
2257  * fencing event is received or a CIB diff occurrs.
2258  */
2259 static void
2260 refresh_after_event(gboolean data_updated, gboolean enforce)
     /*  */
2261 {
2262     static int updates = 0;
2263     time_t now = time(NULL);
2264 
2265     if (data_updated) {
2266         updates++;
2267     }
2268 
2269     if(refresh_timer == NULL) {
2270         refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
2271     }
2272 
2273     if (reconnect_timer > 0) {
2274         /* we will receive a refresh request after successful reconnect */
2275         mainloop_timer_stop(refresh_timer);
2276         return;
2277     }
2278 
2279     /* as we're not handling initial failure of fencer-connection as
2280      * fatal give it a retry here
2281      * not getting here if cib-reconnection is already on the way
2282      */
2283     fencing_connect();
2284 
2285     if (enforce ||
2286         ((now - last_refresh) > (options.reconnect_ms / 1000)) ||
2287         updates >= 10) {
2288         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
2289         mainloop_timer_stop(refresh_timer);
2290         updates = 0;
2291 
2292     } else {
2293         mainloop_timer_start(refresh_timer);
2294     }
2295 }
2296 
2297 /* This function is called for fencing events (see fencing_connect for which ones) when
2298  * --watch-fencing is NOT used on the command line.
2299  */
2300 static void
2301 mon_st_callback_display(stonith_t * st, stonith_event_t * e)
     /*  */
2302 {
2303     if (st->state == stonith_disconnected) {
2304         /* disconnect cib as well and have everything reconnect */
2305         mon_cib_connection_destroy(NULL);
2306     } else {
2307         out->progress(out, false);
2308         refresh_after_event(TRUE, FALSE);
2309     }
2310 }
2311 
2312 static void
2313 clean_up_cib_connection(void)
     /*  */
2314 {
2315     if (cib == NULL) {
2316         return;
2317     }
2318 
2319     cib->cmds->signoff(cib);
2320     cib_delete(cib);
2321     cib = NULL;
2322 }
2323 
2324 static void
2325 clean_up_fencing_connection(void)
     /*  */
2326 {
2327     if (st == NULL) {
2328         return;
2329     }
2330 
2331     if (st->state != stonith_disconnected) {
2332         st->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT);
2333         st->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE);
2334         st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY);
2335         st->cmds->disconnect(st);
2336     }
2337 
2338     stonith_api_delete(st);
2339     st = NULL;
2340 }
2341 
2342 /*
2343  * De-init ncurses, disconnect from the CIB manager, disconnect fencing,
2344  * deallocate memory and show usage-message if requested.
2345  *
2346  * We don't actually return, but nominally returning crm_exit_t allows a usage
2347  * like "return clean_up(exit_code);" which helps static analysis understand the
2348  * code flow.
2349  */
2350 static crm_exit_t
2351 clean_up(crm_exit_t exit_code)
     /*  */
2352 {
2353     /* Quitting crm_mon is much more complicated than it ought to be. */
2354 
2355     /* (1) Close connections, free things, etc. */
2356     clean_up_cib_connection();
2357     clean_up_fencing_connection();
2358     free(options.neg_location_prefix);
2359     free(options.only_node);
2360     free(options.only_rsc);
2361     free(options.pid_file);
2362     g_slist_free_full(options.includes_excludes, free);
2363 
2364     pe_free_working_set(mon_data_set);
2365     mon_data_set = NULL;
2366 
2367     g_strfreev(processed_args);
2368 
2369     /* (2) If this is abnormal termination and we're in curses mode, shut down
2370      * curses first.  Any messages displayed to the screen before curses is shut
2371      * down will be lost because doing the shut down will also restore the
2372      * screen to whatever it looked like before crm_mon was started.
2373      */
2374     if ((error != NULL || exit_code == CRM_EX_USAGE) && output_format == mon_output_console) {
2375         out->finish(out, exit_code, false, NULL);
2376         pcmk__output_free(out);
2377         out = NULL;
2378     }
2379 
2380     /* (3) If this is a command line usage related failure, print the usage
2381      * message.
2382      */
2383     if (exit_code == CRM_EX_USAGE && (output_format == mon_output_console || output_format == mon_output_plain)) {
2384         char *help = g_option_context_get_help(context, TRUE, NULL);
2385 
2386         fprintf(stderr, "%s", help);
2387         g_free(help);
2388     }
2389 
2390     pcmk__free_arg_context(context);
2391 
2392     /* (4) If this is any kind of error, print the error out and exit.  Make
2393      * sure to handle situations both before and after formatted output is
2394      * set up.  We want errors to appear formatted if at all possible.
2395      */
2396     if (error != NULL) {
2397         if (out != NULL) {
2398             out->err(out, "%s: %s", g_get_prgname(), error->message);
2399             out->finish(out, exit_code, true, NULL);
2400             pcmk__output_free(out);
2401         } else {
2402             fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
2403         }
2404 
2405         g_clear_error(&error);
2406         crm_exit(exit_code);
2407     }
2408 
2409     /* (5) Print formatted output to the screen if we made it far enough in
2410      * crm_mon to be able to do so.
2411      */
2412     if (out != NULL) {
2413         if (!options.daemonize) {
2414             out->finish(out, exit_code, true, NULL);
2415         }
2416 
2417         pcmk__output_free(out);
2418         pcmk__unregister_formats();
2419     }
2420 
2421     crm_exit(exit_code);
2422 }