root/tools/crm_mon.c

/* */

DEFINITIONS

1. all_includes
2. default_includes
3. find_section_bit
4. apply_exclude
5. apply_include
6. apply_include_exclude
7. user_include_exclude_cb
8. include_exclude_cb
9. as_cgi_cb
10. as_html_cb
11. as_simple_cb
12. as_xml_cb
13. fence_history_cb
14. group_by_node_cb
15. hide_headers_cb
16. inactive_resources_cb
17. no_curses_cb
18. print_brief_cb
19. print_detail_cb
20. print_timing_cb
21. reconnect_cb
22. show_attributes_cb
23. show_bans_cb
24. show_failcounts_cb
25. show_operations_cb
26. show_tickets_cb
27. use_cib_file_cb
28. reconnect_after_timeout
29. mon_cib_connection_destroy
30. mon_shutdown
31. mon_winresize
32. fencing_connect
33. cib_connect
34. set_fencing_options
35. pacemakerd_event_cb
36. pacemakerd_status
37. get_option_desc
38. detect_user_input
39. avoid_zombies
40. build_arg_context
41. add_output_args
42. reconcile_output_format
43. handle_connection_failures
44. one_shot
45. main
46. print_simple_status
47. send_custom_trap
48. handle_rsc_op
49. mon_trigger_refresh
50. crm_diff_update_v2
51. crm_diff_update_v1
52. crm_diff_update
53. get_fencing_history
54. mon_refresh_display
55. mon_st_callback_event
56. refresh_after_event
57. mon_st_callback_display
58. clean_up_cib_connection
59. clean_up_fencing_connection
60. clean_up

   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 
  14 #include <crm/crm.h>
  15 
  16 #include <stdio.h>
  17 #include <sys/types.h>
  18 #include <sys/stat.h>
  19 #include <unistd.h>
  20 
  21 #include <stdlib.h>
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <libgen.h>
  25 #include <signal.h>
  26 #include <sys/utsname.h>
  27 
  28 #include <crm/msg_xml.h>
  29 #include <crm/services.h>
  30 #include <crm/lrmd.h>
  31 #include <crm/common/cmdline_internal.h>
  32 #include <crm/common/curses_internal.h>
  33 #include <crm/common/internal.h>  // pcmk__ends_with_ext()
  34 #include <crm/common/ipc.h>
  35 #include <crm/common/iso8601_internal.h>
  36 #include <crm/common/mainloop.h>
  37 #include <crm/common/output.h>
  38 #include <crm/common/output_internal.h>
  39 #include <crm/common/util.h>
  40 #include <crm/common/xml.h>
  41 #include <crm/common/xml_internal.h>
  42 
  43 #include <crm/cib/internal.h>
  44 #include <crm/pengine/status.h>
  45 #include <crm/pengine/internal.h>
  46 #include <pacemaker-internal.h>
  47 #include <crm/stonith-ng.h>
  48 #include <crm/fencing/internal.h>
  49 
  50 #include "crm_mon.h"
  51 
  52 #define SUMMARY "Provides a summary of cluster's current state.\n\n" \
  53                 "Outputs varying levels of detail in a number of different formats."
  54 
  55 /*
  56  * Definitions indicating which items to print
  57  */
  58 
  59 static unsigned int show;
  60 static unsigned int show_opts = pcmk_show_pending;
  61 
  62 /*
  63  * Definitions indicating how to output
  64  */
  65 
  66 static mon_output_format_t output_format = mon_output_unset;
  67 
  68 /* other globals */
  69 static GIOChannel *io_channel = NULL;
  70 static GMainLoop *mainloop = NULL;
  71 static guint reconnect_timer = 0;
  72 static mainloop_timer_t *refresh_timer = NULL;
  73 static pe_working_set_t *mon_data_set = NULL;
  74 
  75 static cib_t *cib = NULL;
  76 static stonith_t *st = NULL;
  77 static xmlNode *current_cib = NULL;
  78 
  79 static GError *error = NULL;
  80 static pcmk__common_args_t *args = NULL;
  81 static pcmk__output_t *out = NULL;
  82 static GOptionContext *context = NULL;
  83 static gchar **processed_args = NULL;
  84 
  85 static time_t last_refresh = 0;
  86 volatile crm_trigger_t *refresh_trigger = NULL;
  87 
  88 static gboolean fence_history = FALSE;
  89 static gboolean has_warnings = FALSE;
  90 static gboolean on_remote_node = FALSE;
  91 static gboolean use_cib_native = FALSE;
  92 
  93 int interactive_fence_level = 0;
  94 
  95 static pcmk__supported_format_t formats[] = {
  96 #if CURSES_ENABLED
  97     CRM_MON_SUPPORTED_FORMAT_CURSES,
  98 #endif
  99     PCMK__SUPPORTED_FORMAT_HTML,
 100     PCMK__SUPPORTED_FORMAT_NONE,
 101     PCMK__SUPPORTED_FORMAT_TEXT,
 102     PCMK__SUPPORTED_FORMAT_XML,
 103     { NULL, NULL, NULL }
 104 };
 105 
 106 /* Define exit codes for monitoring-compatible output
 107  * For nagios plugins, the possibilities are
 108  * OK=0, WARN=1, CRIT=2, and UNKNOWN=3
 109  */
 110 #define MON_STATUS_WARN    CRM_EX_ERROR
 111 #define MON_STATUS_CRIT    CRM_EX_INVALID_PARAM
 112 #define MON_STATUS_UNKNOWN CRM_EX_UNIMPLEMENT_FEATURE
 113 
 114 #define RECONNECT_MSECS 5000
 115 
 116 struct {
 117     guint reconnect_ms;
 118     gboolean daemonize;
 119     gboolean fence_connect;
 120     gboolean one_shot;
 121     gboolean print_pending;
 122     gboolean show_bans;
 123     gboolean watch_fencing;
 124     char *pid_file;
 125     char *external_agent;
 126     char *external_recipient;
 127     char *neg_location_prefix;
 128     char *only_node;
 129     char *only_rsc;
 130     GSList *user_includes_excludes;
 131     GSList *includes_excludes;
 132 } options = {
 133     .fence_connect = TRUE,
 134     .reconnect_ms = RECONNECT_MSECS
 135 };
 136 
 137 static void clean_up_cib_connection(void);
 138 static void clean_up_fencing_connection(void);
 139 static crm_exit_t clean_up(crm_exit_t exit_code);
 140 static void crm_diff_update(const char *event, xmlNode * msg);
 141 static void handle_connection_failures(int rc);
 142 static int mon_refresh_display(gpointer user_data);
 143 static int cib_connect(gboolean full);
 144 static int fencing_connect(void);
 145 static int pacemakerd_status(void);
 146 static void mon_st_callback_event(stonith_t * st, stonith_event_t * e);
 147 static void mon_st_callback_display(stonith_t * st, stonith_event_t * e);
 148 static void refresh_after_event(gboolean data_updated, gboolean enforce);
 149 
 150 static unsigned int
 151 all_includes(mon_output_format_t fmt) {
     /*  */
 152     if (fmt == mon_output_monitor || fmt == mon_output_plain || fmt == mon_output_console) {
 153         return ~pcmk_section_options;
 154     } else {
 155         return pcmk_section_all;
 156     }
 157 }
 158 
 159 static unsigned int
 160 default_includes(mon_output_format_t fmt) {
     /*  */
 161     switch (fmt) {
 162         case mon_output_monitor:
 163         case mon_output_plain:
 164         case mon_output_console:
 165             return pcmk_section_stack | pcmk_section_dc | pcmk_section_times | pcmk_section_counts |
 166                    pcmk_section_nodes | pcmk_section_resources | pcmk_section_failures;
 167 
 168         case mon_output_xml:
 169         case mon_output_legacy_xml:
 170             return all_includes(fmt);
 171 
 172         case mon_output_html:
 173         case mon_output_cgi:
 174             return pcmk_section_summary | pcmk_section_nodes | pcmk_section_resources |
 175                    pcmk_section_failures;
 176 
 177         default:
 178             return 0;
 179     }
 180 }
 181 
 182 struct {
 183     const char *name;
 184     unsigned int bit;
 185 } sections[] = {
 186     { "attributes", pcmk_section_attributes },
 187     { "bans", pcmk_section_bans },
 188     { "counts", pcmk_section_counts },
 189     { "dc", pcmk_section_dc },
 190     { "failcounts", pcmk_section_failcounts },
 191     { "failures", pcmk_section_failures },
 192     { "fencing", pcmk_section_fencing_all },
 193     { "fencing-failed", pcmk_section_fence_failed },
 194     { "fencing-pending", pcmk_section_fence_pending },
 195     { "fencing-succeeded", pcmk_section_fence_worked },
 196     { "nodes", pcmk_section_nodes },
 197     { "operations", pcmk_section_operations },
 198     { "options", pcmk_section_options },
 199     { "resources", pcmk_section_resources },
 200     { "stack", pcmk_section_stack },
 201     { "summary", pcmk_section_summary },
 202     { "tickets", pcmk_section_tickets },
 203     { "times", pcmk_section_times },
 204     { NULL }
 205 };
 206 
 207 static unsigned int
 208 find_section_bit(const char *name) {
     /*  */
 209     for (int i = 0; sections[i].name != NULL; i++) {
 210         if (pcmk__str_eq(sections[i].name, name, pcmk__str_casei)) {
 211             return sections[i].bit;
 212         }
 213     }
 214 
 215     return 0;
 216 }
 217 
 218 static gboolean
 219 apply_exclude(const gchar *excludes, GError **error) {
     /*  */
 220     char **parts = NULL;
 221     gboolean result = TRUE;
 222 
 223     parts = g_strsplit(excludes, ",", 0);
 224     for (char **s = parts; *s != NULL; s++) {
 225         unsigned int bit = find_section_bit(*s);
 226 
 227         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 228             show = 0;
 229         } else if (pcmk__str_eq(*s, "none", pcmk__str_none)) {
 230             show = all_includes(output_format);
 231         } else if (bit != 0) {
 232             show &= ~bit;
 233         } else {
 234             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 235                         "--exclude options: all, attributes, bans, counts, dc, "
 236                         "failcounts, failures, fencing, fencing-failed, "
 237                         "fencing-pending, fencing-succeeded, nodes, none, "
 238                         "operations, options, resources, stack, summary, "
 239                         "tickets, times");
 240             result = FALSE;
 241             break;
 242         }
 243     }
 244     g_strfreev(parts);
 245     return result;
 246 }
 247 
 248 static gboolean
 249 apply_include(const gchar *includes, GError **error) {
     /*  */
 250     char **parts = NULL;
 251     gboolean result = TRUE;
 252 
 253     parts = g_strsplit(includes, ",", 0);
 254     for (char **s = parts; *s != NULL; s++) {
 255         unsigned int bit = find_section_bit(*s);
 256 
 257         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 258             show = all_includes(output_format);
 259         } else if (pcmk__starts_with(*s, "bans")) {
 260             show |= pcmk_section_bans;
 261             if (options.neg_location_prefix != NULL) {
 262                 free(options.neg_location_prefix);
 263                 options.neg_location_prefix = NULL;
 264             }
 265 
 266             if (strlen(*s) > 4 && (*s)[4] == ':') {
 267                 options.neg_location_prefix = strdup(*s+5);
 268             }
 269         } else if (pcmk__str_any_of(*s, "default", "defaults", NULL)) {
 270             show |= default_includes(output_format);
 271         } else if (pcmk__str_eq(*s, "none", pcmk__str_none)) {
 272             show = 0;
 273         } else if (bit != 0) {
 274             show |= bit;
 275         } else {
 276             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 277                         "--include options: all, attributes, bans[:PREFIX], counts, dc, "
 278                         "default, failcounts, failures, fencing, fencing-failed, "
 279                         "fencing-pending, fencing-succeeded, nodes, none, operations, "
 280                         "options, resources, stack, summary, tickets, times");
 281             result = FALSE;
 282             break;
 283         }
 284     }
 285     g_strfreev(parts);
 286     return result;
 287 }
 288 
 289 static gboolean
 290 apply_include_exclude(GSList *lst, mon_output_format_t fmt, GError **error) {
     /*  */
 291     gboolean rc = TRUE;
 292     GSList *node = lst;
 293 
 294     /* Set the default of what to display here.  Note that we OR everything to
 295      * show instead of set show directly because it could have already had some
 296      * settings applied to it in main.
 297      */
 298     show |= default_includes(fmt);
 299 
 300     while (node != NULL) {
 301         char *s = node->data;
 302 
 303         if (pcmk__starts_with(s, "--include=")) {
 304             rc = apply_include(s+10, error);
 305         } else if (pcmk__starts_with(s, "-I=")) {
 306             rc = apply_include(s+3, error);
 307         } else if (pcmk__starts_with(s, "--exclude=")) {
 308             rc = apply_exclude(s+10, error);
 309         } else if (pcmk__starts_with(s, "-U=")) {
 310             rc = apply_exclude(s+3, error);
 311         }
 312 
 313         if (rc != TRUE) {
 314             break;
 315         }
 316 
 317         node = node->next;
 318     }
 319 
 320     return rc;
 321 }
 322 
 323 static gboolean
 324 user_include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 325     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 326 
 327     options.user_includes_excludes = g_slist_append(options.user_includes_excludes, s);
 328     return TRUE;
 329 }
 330 
 331 static gboolean
 332 include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 333     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 334 
 335     options.includes_excludes = g_slist_append(options.includes_excludes, s);
 336     return TRUE;
 337 }
 338 
 339 static gboolean
 340 as_cgi_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 341     if (args->output_ty != NULL) {
 342         free(args->output_ty);
 343     }
 344 
 345     args->output_ty = strdup("html");
 346     output_format = mon_output_cgi;
 347     options.one_shot = TRUE;
 348     return TRUE;
 349 }
 350 
 351 static gboolean
 352 as_html_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 353     if (args->output_ty != NULL) {
 354         free(args->output_ty);
 355     }
 356 
 357     if (args->output_dest != NULL) {
 358         free(args->output_dest);
 359         args->output_dest = NULL;
 360     }
 361 
 362     if (optarg != NULL) {
 363         args->output_dest = strdup(optarg);
 364     }
 365 
 366     args->output_ty = strdup("html");
 367     output_format = mon_output_html;
 368     umask(S_IWGRP | S_IWOTH);
 369     return TRUE;
 370 }
 371 
 372 static gboolean
 373 as_simple_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 374     if (args->output_ty != NULL) {
 375         free(args->output_ty);
 376     }
 377 
 378     args->output_ty = strdup("text");
 379     output_format = mon_output_monitor;
 380     options.one_shot = TRUE;
 381     return TRUE;
 382 }
 383 
 384 static gboolean
 385 as_xml_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 386     if (args->output_ty != NULL) {
 387         free(args->output_ty);
 388     }
 389 
 390     args->output_ty = strdup("xml");
 391     output_format = mon_output_legacy_xml;
 392     return TRUE;
 393 }
 394 
 395 static gboolean
 396 fence_history_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 397     if (optarg == NULL) {
 398         interactive_fence_level = 2;
 399     } else {
 400         pcmk__scan_min_int(optarg, &interactive_fence_level, 0);
 401     }
 402 
 403     switch (interactive_fence_level) {
 404         case 3:
 405             options.fence_connect = TRUE;
 406             fence_history = TRUE;
 407             return include_exclude_cb("--include", "fencing", data, err);
 408 
 409         case 2:
 410             options.fence_connect = TRUE;
 411             fence_history = TRUE;
 412             return include_exclude_cb("--include", "fencing", data, err);
 413 
 414         case 1:
 415             options.fence_connect = TRUE;
 416             fence_history = TRUE;
 417             return include_exclude_cb("--include", "fencing-failed,fencing-pending", data, err);
 418 
 419         case 0:
 420             options.fence_connect = FALSE;
 421             fence_history = FALSE;
 422             return include_exclude_cb("--exclude", "fencing", data, err);
 423 
 424         default:
 425             g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Fence history must be 0-3");
 426             return FALSE;
 427     }
 428 }
 429 
 430 static gboolean
 431 group_by_node_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 432     show_opts |= pcmk_show_rscs_by_node;
 433     return TRUE;
 434 }
 435 
 436 static gboolean
 437 hide_headers_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 438     return include_exclude_cb("--exclude", "summary", data, err);
 439 }
 440 
 441 static gboolean
 442 inactive_resources_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 443     show_opts |= pcmk_show_inactive_rscs;
 444     return TRUE;
 445 }
 446 
 447 static gboolean
 448 no_curses_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 449     output_format = mon_output_plain;
 450     return TRUE;
 451 }
 452 
 453 static gboolean
 454 print_brief_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 455     show_opts |= pcmk_show_brief;
 456     return TRUE;
 457 }
 458 
 459 static gboolean
 460 print_detail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 461     show_opts |= pcmk_show_details;
 462     return TRUE;
 463 }
 464 
 465 static gboolean
 466 print_timing_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 467     show_opts |= pcmk_show_timing;
 468     return include_exclude_cb("--include", "operations", data, err);
 469 }
 470 
 471 static gboolean
 472 reconnect_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 473     int rc = crm_get_msec(optarg);
 474 
 475     if (rc == -1) {
 476         g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid value for -i: %s", optarg);
 477         return FALSE;
 478     } else {
 479         options.reconnect_ms = crm_parse_interval_spec(optarg);
 480     }
 481 
 482     return TRUE;
 483 }
 484 
 485 static gboolean
 486 show_attributes_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 487     return include_exclude_cb("--include", "attributes", data, err);
 488 }
 489 
 490 static gboolean
 491 show_bans_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 492     if (optarg != NULL) {
 493         char *s = crm_strdup_printf("bans:%s", optarg);
 494         gboolean rc = include_exclude_cb("--include", s, data, err);
 495         free(s);
 496         return rc;
 497     } else {
 498         return include_exclude_cb("--include", "bans", data, err);
 499     }
 500 }
 501 
 502 static gboolean
 503 show_failcounts_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 504     return include_exclude_cb("--include", "failcounts", data, err);
 505 }
 506 
 507 static gboolean
 508 show_operations_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 509     return include_exclude_cb("--include", "failcounts,operations", data, err);
 510 }
 511 
 512 static gboolean
 513 show_tickets_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 514     return include_exclude_cb("--include", "tickets", data, err);
 515 }
 516 
 517 static gboolean
 518 use_cib_file_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 519     setenv("CIB_file", optarg, 1);
 520     options.one_shot = TRUE;
 521     return TRUE;
 522 }
 523 
 524 #define INDENT "                                    "
 525 
 526 /* *INDENT-OFF* */
 527 static GOptionEntry addl_entries[] = {
 528     { "interval", 'i', 0, G_OPTION_ARG_CALLBACK, reconnect_cb,
 529       "Update frequency (default is 5 seconds)",
 530       "TIMESPEC" },
 531 
 532     { "one-shot", '1', 0, G_OPTION_ARG_NONE, &options.one_shot,
 533       "Display the cluster status once on the console and exit",
 534       NULL },
 535 
 536     { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &options.daemonize,
 537       "Run in the background as a daemon.\n"
 538       INDENT "Requires at least one of --output-to and --external-agent.",
 539       NULL },
 540 
 541     { "pid-file", 'p', 0, G_OPTION_ARG_FILENAME, &options.pid_file,
 542       "(Advanced) Daemon pid file location",
 543       "FILE" },
 544 
 545     { "external-agent", 'E', 0, G_OPTION_ARG_FILENAME, &options.external_agent,
 546       "A program to run when resource operations take place",
 547       "FILE" },
 548 
 549     { "external-recipient", 'e', 0, G_OPTION_ARG_STRING, &options.external_recipient,
 550       "A recipient for your program (assuming you want the program to send something to someone).",
 551       "RCPT" },
 552 
 553     { "watch-fencing", 'W', 0, G_OPTION_ARG_NONE, &options.watch_fencing,
 554       "Listen for fencing events. For use with --external-agent.",
 555       NULL },
 556 
 557     { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, use_cib_file_cb,
 558       NULL,
 559       NULL },
 560 
 561     { NULL }
 562 };
 563 
 564 static GOptionEntry display_entries[] = {
 565     { "include", 'I', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 566       "A list of sections to include in the output.\n"
 567       INDENT "See `Output Control` help for more information.",
 568       "SECTION(s)" },
 569 
 570     { "exclude", 'U', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 571       "A list of sections to exclude from the output.\n"
 572       INDENT "See `Output Control` help for more information.",
 573       "SECTION(s)" },
 574 
 575     { "node", 0, 0, G_OPTION_ARG_STRING, &options.only_node,
 576       "When displaying information about nodes, show only what's related to the given\n"
 577       INDENT "node, or to all nodes tagged with the given tag",
 578       "NODE" },
 579 
 580     { "resource", 0, 0, G_OPTION_ARG_STRING, &options.only_rsc,
 581       "When displaying information about resources, show only what's related to the given\n"
 582       INDENT "resource, or to all resources tagged with the given tag",
 583       "RSC" },
 584 
 585     { "group-by-node", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, group_by_node_cb,
 586       "Group resources by node",
 587       NULL },
 588 
 589     { "inactive", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, inactive_resources_cb,
 590       "Display inactive resources",
 591       NULL },
 592 
 593     { "failcounts", 'f', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_failcounts_cb,
 594       "Display resource fail counts",
 595       NULL },
 596 
 597     { "operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_operations_cb,
 598       "Display resource operation history",
 599       NULL },
 600 
 601     { "timing-details", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_timing_cb,
 602       "Display resource operation history with timing details",
 603       NULL },
 604 
 605     { "tickets", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_tickets_cb,
 606       "Display cluster tickets",
 607       NULL },
 608 
 609     { "fence-history", 'm', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, fence_history_cb,
 610       "Show fence history:\n"
 611       INDENT "0=off, 1=failures and pending (default without option),\n"
 612       INDENT "2=add successes (default without value for option),\n"
 613       INDENT "3=show full history without reduction to most recent of each flavor",
 614       "LEVEL" },
 615 
 616     { "neg-locations", 'L', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, show_bans_cb,
 617       "Display negative location constraints [optionally filtered by id prefix]",
 618       NULL },
 619 
 620     { "show-node-attributes", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_attributes_cb,
 621       "Display node attributes",
 622       NULL },
 623 
 624     { "hide-headers", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, hide_headers_cb,
 625       "Hide all headers",
 626       NULL },
 627 
 628     { "show-detail", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_detail_cb,
 629       "Show more details (node IDs, individual clone instances)",
 630       NULL },
 631 
 632     { "brief", 'b', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_brief_cb,
 633       "Brief output",
 634       NULL },
 635 
 636     { "pending", 'j', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.print_pending,
 637       "Display pending state if 'record-pending' is enabled",
 638       NULL },
 639 
 640     { "simple-status", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_simple_cb,
 641       "Display the cluster status once as a simple one line output (suitable for nagios)",
 642       NULL },
 643 
 644     { NULL }
 645 };
 646 
 647 static GOptionEntry deprecated_entries[] = {
 648     { "as-html", 'h', G_OPTION_FLAG_FILENAME, G_OPTION_ARG_CALLBACK, as_html_cb,
 649       "Write cluster status to the named HTML file.\n"
 650       INDENT "Use --output-as=html --output-to=FILE instead.",
 651       "FILE" },
 652 
 653     { "as-xml", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_xml_cb,
 654       "Write cluster status as XML to stdout. This will enable one-shot mode.\n"
 655       INDENT "Use --output-as=xml instead.",
 656       NULL },
 657 
 658     { "disable-ncurses", 'N', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, no_curses_cb,
 659       "Disable the use of ncurses.\n"
 660       INDENT "Use --output-as=text instead.",
 661       NULL },
 662 
 663     { "web-cgi", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_cgi_cb,
 664       "Web mode with output suitable for CGI (preselected when run as *.cgi).\n"
 665       INDENT "Use --output-as=html --html-cgi instead.",
 666       NULL },
 667 
 668     { NULL }
 669 };
 670 /* *INDENT-ON* */
 671 
 672 /* Reconnect to the CIB and fencing agent after reconnect_ms has passed.  This sounds
 673  * like it would be more broadly useful, but only ever happens after a disconnect via
 674  * mon_cib_connection_destroy.
 675  */
 676 static gboolean
 677 reconnect_after_timeout(gpointer data)
     /*  */
 678 {
 679 #if CURSES_ENABLED
 680     if (output_format == mon_output_console) {
 681         clear();
 682         refresh();
 683     }
 684 #endif
 685 
 686     out->info(out, "Reconnecting...");
 687     if (pacemakerd_status() == pcmk_rc_ok) {
 688         fencing_connect();
 689         if (cib_connect(TRUE) == pcmk_rc_ok) {
 690             /* trigger redrawing the screen (needs reconnect_timer == 0) */
 691             reconnect_timer = 0;
 692             refresh_after_event(FALSE, TRUE);
 693             return G_SOURCE_REMOVE;
 694         }
 695     }
 696 
 697     reconnect_timer = g_timeout_add(options.reconnect_ms,
 698                                     reconnect_after_timeout, NULL);
 699     return G_SOURCE_REMOVE;
 700 }
 701 
 702 /* Called from various places when we are disconnected from the CIB or from the
 703  * fencing agent.  If the CIB connection is still valid, this function will also
 704  * attempt to sign off and reconnect.
 705  */
 706 static void
 707 mon_cib_connection_destroy(gpointer user_data)
     /*  */
 708 {
 709     out->info(out, "Connection to the cluster-daemons terminated");
 710 
 711     if (refresh_timer != NULL) {
 712         /* we'll trigger a refresh after reconnect */
 713         mainloop_timer_stop(refresh_timer);
 714     }
 715     if (reconnect_timer) {
 716         /* we'll trigger a new reconnect-timeout at the end */
 717         g_source_remove(reconnect_timer);
 718         reconnect_timer = 0;
 719     }
 720     if (st) {
 721         /* the client API won't properly reconnect notifications
 722          * if they are still in the table - so remove them
 723          */
 724         clean_up_fencing_connection();
 725     }
 726     if (cib) {
 727         cib->cmds->signoff(cib);
 728         reconnect_timer = g_timeout_add(options.reconnect_ms,
 729                                         reconnect_after_timeout, NULL);
 730     }
 731     return;
 732 }
 733 
 734 /* Signal handler installed into the mainloop for normal program shutdown */
 735 static void
 736 mon_shutdown(int nsig)
     /*  */
 737 {
 738     clean_up(CRM_EX_OK);
 739 }
 740 
 741 #if CURSES_ENABLED
 742 static volatile sighandler_t ncurses_winch_handler;
 743 
 744 /* Signal handler installed the regular way (not into the main loop) for when
 745  * the screen is resized.  Commonly, this happens when running in an xterm and
 746  * the user changes its size.
 747  */
 748 static void
 749 mon_winresize(int nsig)
     /*  */
 750 {
 751     static int not_done;
 752     int lines = 0, cols = 0;
 753 
 754     if (!not_done++) {
 755         if (ncurses_winch_handler)
 756             /* the original ncurses WINCH signal handler does the
 757              * magic of retrieving the new window size;
 758              * otherwise, we'd have to use ioctl or tgetent */
 759             (*ncurses_winch_handler) (SIGWINCH);
 760         getmaxyx(stdscr, lines, cols);
 761         resizeterm(lines, cols);
 762         /* Alert the mainloop code we'd like the refresh_trigger to run next
 763          * time the mainloop gets around to checking.
 764          */
 765         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
 766     }
 767     not_done--;
 768 }
 769 #endif
 770 
 771 static int
 772 fencing_connect(void)
     /*  */
 773 {
 774     int rc = pcmk_ok;
 775 
 776     if (options.fence_connect && st == NULL) {
 777         st = stonith_api_new();
 778     }
 779 
 780     if (!options.fence_connect || st == NULL || st->state != stonith_disconnected) {
 781         return rc;
 782     }
 783 
 784     rc = st->cmds->connect(st, crm_system_name, NULL);
 785     if (rc == pcmk_ok) {
 786         crm_trace("Setting up stonith callbacks");
 787         if (options.watch_fencing) {
 788             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 789                                             mon_st_callback_event);
 790             st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback_event);
 791         } else {
 792             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 793                                             mon_st_callback_display);
 794             st->cmds->register_notification(st, T_STONITH_NOTIFY_HISTORY, mon_st_callback_display);
 795         }
 796     } else {
 797         clean_up_fencing_connection();
 798     }
 799 
 800     return rc;
 801 }
 802 
 803 static int
 804 cib_connect(gboolean full)
     /*  */
 805 {
 806     int rc = pcmk_rc_ok;
 807 
 808     CRM_CHECK(cib != NULL, return EINVAL);
 809 
 810     if (cib->state == cib_connected_query ||
 811         cib->state == cib_connected_command) {
 812         return rc;
 813     }
 814 
 815     crm_trace("Connecting to the CIB");
 816 
 817     rc = pcmk_legacy2rc(cib->cmds->signon(cib, crm_system_name, cib_query));
 818     if (rc != pcmk_rc_ok) {
 819         out->err(out, "Could not connect to the CIB: %s",
 820                  pcmk_rc_str(rc));
 821         return rc;
 822     }
 823 
 824 #if CURSES_ENABLED
 825     /* just show this if refresh is gonna remove all traces */
 826     if (output_format == mon_output_console) {
 827         out->info(out,"Waiting for CIB ...");
 828     }
 829 #endif
 830 
 831     rc = pcmk_legacy2rc(cib->cmds->query(cib, NULL, &current_cib,
 832                                          cib_scope_local | cib_sync_call));
 833 
 834     if (rc == pcmk_rc_ok && full) {
 835         rc = pcmk_legacy2rc(cib->cmds->set_connection_dnotify(cib,
 836             mon_cib_connection_destroy));
 837         if (rc == EPROTONOSUPPORT) {
 838             out->err(out,
 839                      "Notification setup not supported, won't be "
 840                      "able to reconnect after failure");
 841             if (output_format == mon_output_console) {
 842                 sleep(2);
 843             }
 844             rc = pcmk_rc_ok;
 845         }
 846 
 847         if (rc == pcmk_rc_ok) {
 848             cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY,
 849                                            crm_diff_update);
 850             rc = pcmk_legacy2rc(cib->cmds->add_notify_callback(cib,
 851                                     T_CIB_DIFF_NOTIFY, crm_diff_update));
 852         }
 853 
 854         if (rc != pcmk_rc_ok) {
 855             out->err(out, "Notification setup failed, could not monitor CIB actions");
 856             clean_up_cib_connection();
 857             clean_up_fencing_connection();
 858         }
 859     }
 860     return rc;
 861 }
 862 
 863 /* This is used to set up the fencing options after the interactive UI has been stared.
 864  * fence_history_cb can't be used because it builds up a list of includes/excludes that
 865  * then have to be processed with apply_include_exclude and that could affect other
 866  * things.
 867  */
 868 static void
 869 set_fencing_options(int level)
     /*  */
 870 {
 871     switch (level) {
 872         case 3:
 873             options.fence_connect = TRUE;
 874             fence_history = TRUE;
 875             show |= pcmk_section_fencing_all;
 876             break;
 877 
 878         case 2:
 879             options.fence_connect = TRUE;
 880             fence_history = TRUE;
 881             show |= pcmk_section_fencing_all;
 882             break;
 883 
 884         case 1:
 885             options.fence_connect = TRUE;
 886             fence_history = TRUE;
 887             show |= pcmk_section_fence_failed | pcmk_section_fence_pending;
 888             break;
 889 
 890         default:
 891             interactive_fence_level = 0;
 892             options.fence_connect = FALSE;
 893             fence_history = FALSE;
 894             show &= ~pcmk_section_fencing_all;
 895             break;
 896     }
 897 }
 898 
 899 /* Before trying to connect to fencer or cib check for state of
 900    pacemakerd - just no sense in trying till pacemakerd has
 901    taken care of starting all the sub-processes
 902 
 903    Only noteworthy thing to show here is when pacemakerd is
 904    waiting for startup-trigger from SBD.
 905  */
 906 static void
 907 pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
     /*  */
 908                     enum pcmk_ipc_event event_type, crm_exit_t status,
 909                     void *event_data, void *user_data)
 910 {
 911     pcmk_pacemakerd_api_reply_t *reply = event_data;
 912     enum pcmk_pacemakerd_state *state =
 913         (enum pcmk_pacemakerd_state *) user_data;
 914 
 915     /* we are just interested in the latest reply */
 916     *state = pcmk_pacemakerd_state_invalid;
 917 
 918     switch (event_type) {
 919         case pcmk_ipc_event_reply:
 920             break;
 921 
 922         default:
 923             return;
 924     }
 925 
 926     if (status != CRM_EX_OK) {
 927         out->err(out, "Bad reply from pacemakerd: %s",
 928                  crm_exit_str(status));
 929         return;
 930     }
 931 
 932     if (reply->reply_type != pcmk_pacemakerd_reply_ping) {
 933         out->err(out, "Unknown reply type %d from pacemakerd",
 934                  reply->reply_type);
 935     } else {
 936         if ((reply->data.ping.last_good != (time_t) 0) &&
 937             (reply->data.ping.status == pcmk_rc_ok)) {
 938             *state = reply->data.ping.state;
 939         }
 940     }
 941 }
 942 
 943 static int
 944 pacemakerd_status(void)
     /*  */
 945 {
 946     int rc = pcmk_rc_ok;
 947     pcmk_ipc_api_t *pacemakerd_api = NULL;
 948     enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;
 949 
 950     if (!use_cib_native) {
 951         /* we don't need fully functional pacemakerd otherwise */
 952         return rc;
 953     }
 954     if (cib != NULL &&
 955         (cib->state == cib_connected_query ||
 956          cib->state == cib_connected_command)) {
 957         /* As long as we have a cib-connection let's go with
 958          * that to fetch further cluster-status and avoid
 959          * unnecessary pings to pacemakerd.
 960          * If cluster is going down and fencer is down already
 961          * this will lead to a silently failing fencer reconnect.
 962          * On cluster startup we shouldn't see this situation
 963          * as first we do is wait for pacemakerd to report all
 964          * daemons running.
 965          */
 966         return rc;
 967     }
 968     rc = pcmk_new_ipc_api(&pacemakerd_api, pcmk_ipc_pacemakerd);
 969     if (pacemakerd_api == NULL) {
 970         out->err(out, "Could not connect to pacemakerd: %s",
 971                  pcmk_rc_str(rc));
 972         /* this is unrecoverable so return with rc we have */
 973         return rc;
 974     }
 975     pcmk_register_ipc_callback(pacemakerd_api, pacemakerd_event_cb, (void *) &state);
 976     rc = pcmk_connect_ipc(pacemakerd_api, pcmk_ipc_dispatch_poll);
 977     switch (rc) {
 978         case pcmk_rc_ok:
 979             rc = pcmk_pacemakerd_api_ping(pacemakerd_api, crm_system_name);
 980             if (rc == pcmk_rc_ok) {
 981                 rc = pcmk_poll_ipc(pacemakerd_api, options.reconnect_ms/2);
 982                 if (rc == pcmk_rc_ok) {
 983                     pcmk_dispatch_ipc(pacemakerd_api);
 984                     rc = ENOTCONN;
 985                     if ((output_format == mon_output_console) ||
 986                         (output_format == mon_output_plain)) {
 987                         switch (state) {
 988                             case pcmk_pacemakerd_state_running:
 989                                 rc = pcmk_rc_ok;
 990                                 break;
 991                             case pcmk_pacemakerd_state_starting_daemons:
 992                                 out->info(out,"Pacemaker daemons starting ...");
 993                                 break;
 994                             case pcmk_pacemakerd_state_wait_for_ping:
 995                                 out->info(out,"Waiting for startup-trigger from SBD ...");
 996                                 break;
 997                             case pcmk_pacemakerd_state_shutting_down:
 998                                 out->info(out,"Pacemaker daemons shutting down ...");
 999                                 /* try our luck maybe CIB is still accessible */
1000                                 rc = pcmk_rc_ok;
1001                                 break;
1002                             case pcmk_pacemakerd_state_shutdown_complete:
1003                                 /* assuming pacemakerd doesn't dispatch any pings after entering
1004                                 * that state unless it is waiting for SBD
1005                                 */
1006                                 out->info(out,"Pacemaker daemons shut down - reporting to SBD ...");
1007                                 break;
1008                             default:
1009                                 break;
1010                         }
1011                     } else {
1012                         switch (state) {
1013                             case pcmk_pacemakerd_state_running:
1014                                 rc = pcmk_rc_ok;
1015                                 break;
1016                             case pcmk_pacemakerd_state_shutting_down:
1017                                 /* try our luck maybe CIB is still accessible */
1018                                 rc = pcmk_rc_ok;
1019                                 break;
1020                             default:
1021                                 break;
1022                         }
1023                     }
1024                 }
1025             }
1026             break;
1027         case EREMOTEIO:
1028             rc = pcmk_rc_ok;
1029             on_remote_node = TRUE;
1030 #if CURSES_ENABLED
1031             /* just show this if refresh is gonna remove all traces */
1032             if (output_format == mon_output_console) {
1033                 out->info(out,
1034                     "Running on remote-node waiting to be connected by cluster ...");
1035             }
1036 #endif
1037             break;
1038         default:
1039             break;
1040     }
1041     pcmk_free_ipc_api(pacemakerd_api);
1042     /* returning with ENOTCONN triggers a retry */
1043     return (rc == pcmk_rc_ok)?rc:ENOTCONN;
1044 }
1045 
1046 #if CURSES_ENABLED
1047 static const char *
1048 get_option_desc(char c)
     /*  */
1049 {
1050     const char *desc = "No help available";
1051 
1052     for (GOptionEntry *entry = display_entries; entry != NULL; entry++) {
1053         if (entry->short_name == c) {
1054             desc = entry->description;
1055             break;
1056         }
1057     }
1058     return desc;
1059 }
1060 
1061 #define print_option_help(out, option, condition) \
1062     curses_formatted_printf(out, "%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
1063 
1064 /* This function is called from the main loop when there is something to be read
1065  * on stdin, like an interactive user's keystroke.  All it does is read the keystroke,
1066  * set flags (or show the page showing which keystrokes are valid), and redraw the
1067  * screen.  It does not do anything with connections to the CIB or fencing agent
1068  * agent what would happen in mon_refresh_display.
1069  */
1070 static gboolean
1071 detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer user_data)
     /*  */
1072 {
1073     int c;
1074     gboolean config_mode = FALSE;
1075 
1076     while (1) {
1077 
1078         /* Get user input */
1079         c = getchar();
1080 
1081         switch (c) {
1082             case 'm':
1083                 interactive_fence_level++;
1084                 if (interactive_fence_level > 3) {
1085                     interactive_fence_level = 0;
1086                 }
1087 
1088                 set_fencing_options(interactive_fence_level);
1089                 break;
1090             case 'c':
1091                 show ^= pcmk_section_tickets;
1092                 break;
1093             case 'f':
1094                 show ^= pcmk_section_failcounts;
1095                 break;
1096             case 'n':
1097                 show_opts ^= pcmk_show_rscs_by_node;
1098                 break;
1099             case 'o':
1100                 show ^= pcmk_section_operations;
1101                 if (!pcmk_is_set(show, pcmk_section_operations)) {
1102                     show_opts &= ~pcmk_show_timing;
1103                 }
1104                 break;
1105             case 'r':
1106                 show_opts ^= pcmk_show_inactive_rscs;
1107                 break;
1108             case 'R':
1109                 show_opts ^= pcmk_show_details;
1110                 break;
1111             case 't':
1112                 show_opts ^= pcmk_show_timing;
1113                 if (pcmk_is_set(show_opts, pcmk_show_timing)) {
1114                     show |= pcmk_section_operations;
1115                 }
1116                 break;
1117             case 'A':
1118                 show ^= pcmk_section_attributes;
1119                 break;
1120             case 'L':
1121                 show ^= pcmk_section_bans;
1122                 break;
1123             case 'D':
1124                 /* If any header is shown, clear them all, otherwise set them all */
1125                 if (pcmk_any_flags_set(show, pcmk_section_summary)) {
1126                     show &= ~pcmk_section_summary;
1127                 } else {
1128                     show |= pcmk_section_summary;
1129                 }
1130                 /* Regardless, we don't show options in console mode. */
1131                 show &= ~pcmk_section_options;
1132                 break;
1133             case 'b':
1134                 show_opts ^= pcmk_show_brief;
1135                 break;
1136             case 'j':
1137                 show_opts ^= pcmk_show_pending;
1138                 break;
1139             case '?':
1140                 config_mode = TRUE;
1141                 break;
1142             default:
1143                 /* All other keys just redraw the screen. */
1144                 goto refresh;
1145         }
1146 
1147         if (!config_mode)
1148             goto refresh;
1149 
1150         blank_screen();
1151 
1152         curses_formatted_printf(out, "%s", "Display option change mode\n");
1153         print_option_help(out, 'c', pcmk_is_set(show, pcmk_section_tickets));
1154         print_option_help(out, 'f', pcmk_is_set(show, pcmk_section_failcounts));
1155         print_option_help(out, 'n', pcmk_is_set(show_opts, pcmk_show_rscs_by_node));
1156         print_option_help(out, 'o', pcmk_is_set(show, pcmk_section_operations));
1157         print_option_help(out, 'r', pcmk_is_set(show_opts, pcmk_show_inactive_rscs));
1158         print_option_help(out, 't', pcmk_is_set(show_opts, pcmk_show_timing));
1159         print_option_help(out, 'A', pcmk_is_set(show, pcmk_section_attributes));
1160         print_option_help(out, 'L', pcmk_is_set(show, pcmk_section_bans));
1161         print_option_help(out, 'D', !pcmk_is_set(show, pcmk_section_summary));
1162         print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details));
1163         print_option_help(out, 'b', pcmk_is_set(show_opts, pcmk_show_brief));
1164         print_option_help(out, 'j', pcmk_is_set(show_opts, pcmk_show_pending));
1165         curses_formatted_printf(out, "%d m: \t%s\n", interactive_fence_level, get_option_desc('m'));
1166         curses_formatted_printf(out, "%s", "\nToggle fields via field letter, type any other key to return\n");
1167     }
1168 
1169 refresh:
1170     refresh_after_event(FALSE, TRUE);
1171 
1172     return TRUE;
1173 }
1174 #endif
1175 
1176 // Basically crm_signal_handler(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
1177 static void
1178 avoid_zombies(void)
     /*  */
1179 {
1180     struct sigaction sa;
1181 
1182     memset(&sa, 0, sizeof(struct sigaction));
1183     if (sigemptyset(&sa.sa_mask) < 0) {
1184         crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
1185         return;
1186     }
1187     sa.sa_handler = SIG_IGN;
1188     sa.sa_flags = SA_RESTART|SA_NOCLDWAIT;
1189     if (sigaction(SIGCHLD, &sa, NULL) < 0) {
1190         crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
1191     }
1192 }
1193 
1194 static GOptionContext *
1195 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     /*  */
1196     GOptionContext *context = NULL;
1197 
1198     GOptionEntry extra_prog_entries[] = {
1199         { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
1200           "Be less descriptive in output.",
1201           NULL },
1202 
1203         { NULL }
1204     };
1205 
1206     const char *description = "Notes:\n\n"
1207                               "If this program is called as crm_mon.cgi, --output-as=html --html-cgi will\n"
1208                               "automatically be added to the command line arguments.\n\n"
1209                               "Time Specification:\n\n"
1210                               "The TIMESPEC in any command line option can be specified in many different\n"
1211                               "formats.  It can be just an integer number of seconds, a number plus units\n"
1212                               "(ms/msec/us/usec/s/sec/m/min/h/hr), or an ISO 8601 period specification.\n\n"
1213                               "Output Control:\n\n"
1214                               "By default, a certain list of sections are written to the output destination.\n"
1215                               "The default varies based on the output format - XML includes everything, while\n"
1216                               "other output formats will display less.  This list can be modified with the\n"
1217                               "--include and --exclude command line options.  Each option may be given multiple\n"
1218                               "times on the command line, and each can give a comma-separated list of sections.\n"
1219                               "The options are applied to the default set, from left to right as seen on the\n"
1220                               "command line.  For a list of valid sections, pass --include=list or --exclude=list.\n\n"
1221                               "Interactive Use:\n\n"
1222                               "When run interactively, crm_mon can be told to hide and display various sections\n"
1223                               "of output.  To see a help screen explaining the options, hit '?'.  Any key stroke\n"
1224                               "aside from those listed will cause the screen to refresh.\n\n"
1225                               "Examples:\n\n"
1226                               "Display the cluster status on the console with updates as they occur:\n\n"
1227                               "\tcrm_mon\n\n"
1228                               "Display the cluster status on the console just once then exit:\n\n"
1229                               "\tcrm_mon -1\n\n"
1230                               "Display your cluster status, group resources by node, and include inactive resources in the list:\n\n"
1231                               "\tcrm_mon --group-by-node --inactive\n\n"
1232                               "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:\n\n"
1233                               "\tcrm_mon --daemonize --output-as html --output-to /path/to/docroot/filename.html\n\n"
1234                               "Start crm_mon and export the current cluster status as XML to stdout, then exit:\n\n"
1235                               "\tcrm_mon --output-as xml\n\n";
1236 
1237     context = pcmk__build_arg_context(args, "console (default), html, text, xml", group, NULL);
1238     pcmk__add_main_args(context, extra_prog_entries);
1239     g_option_context_set_description(context, description);
1240 
1241     pcmk__add_arg_group(context, "display", "Display Options:",
1242                         "Show display options", display_entries);
1243     pcmk__add_arg_group(context, "additional", "Additional Options:",
1244                         "Show additional options", addl_entries);
1245     pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
1246                         "Show deprecated options", deprecated_entries);
1247 
1248     return context;
1249 }
1250 
1251 /* If certain format options were specified, we want to set some extra
1252  * options.  We can just process these like they were given on the
1253  * command line.
1254  */
1255 static void
1256 add_output_args(void) {
     /*  */
1257     GError *err = NULL;
1258 
1259     if (output_format == mon_output_plain) {
1260         if (!pcmk__force_args(context, &err, "%s --text-fancy", g_get_prgname())) {
1261             g_propagate_error(&error, err);
1262             clean_up(CRM_EX_USAGE);
1263         }
1264     } else if (output_format == mon_output_cgi) {
1265         if (!pcmk__force_args(context, &err, "%s --html-cgi", g_get_prgname())) {
1266             g_propagate_error(&error, err);
1267             clean_up(CRM_EX_USAGE);
1268         }
1269     } else if (output_format == mon_output_xml) {
1270         if (!pcmk__force_args(context, &err, "%s --xml-simple-list --xml-substitute", g_get_prgname())) {
1271             g_propagate_error(&error, err);
1272             clean_up(CRM_EX_USAGE);
1273         }
1274     } else if (output_format == mon_output_legacy_xml) {
1275         output_format = mon_output_xml;
1276         if (!pcmk__force_args(context, &err, "%s --xml-legacy --xml-substitute", g_get_prgname())) {
1277             g_propagate_error(&error, err);
1278             clean_up(CRM_EX_USAGE);
1279         }
1280     }
1281 }
1282 
1283 /* Which output format to use could come from two places:  The --as-xml
1284  * style arguments we gave in deprecated_entries above, or the formatted output
1285  * arguments added by pcmk__register_formats.  If the latter were used,
1286  * output_format will be mon_output_unset.
1287  *
1288  * Call the callbacks as if those older style arguments were provided so
1289  * the various things they do get done.
1290  */
1291 static void
1292 reconcile_output_format(pcmk__common_args_t *args) {
     /*  */
1293     gboolean retval = TRUE;
1294     GError *err = NULL;
1295 
1296     if (output_format != mon_output_unset) {
1297         return;
1298     }
1299 
1300     if (pcmk__str_eq(args->output_ty, "html", pcmk__str_casei)) {
1301         char *dest = NULL;
1302 
1303         if (args->output_dest != NULL) {
1304             dest = strdup(args->output_dest);
1305         }
1306 
1307         retval = as_html_cb("h", dest, NULL, &err);
1308         free(dest);
1309     } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_casei)) {
1310         retval = no_curses_cb("N", NULL, NULL, &err);
1311     } else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_casei)) {
1312         if (args->output_ty != NULL) {
1313             free(args->output_ty);
1314         }
1315 
1316         args->output_ty = strdup("xml");
1317         output_format = mon_output_xml;
1318     } else if (options.one_shot) {
1319         if (args->output_ty != NULL) {
1320             free(args->output_ty);
1321         }
1322 
1323         args->output_ty = strdup("text");
1324         output_format = mon_output_plain;
1325     } else {
1326         /* Neither old nor new arguments were given, so set the default. */
1327         if (args->output_ty != NULL) {
1328             free(args->output_ty);
1329         }
1330 
1331         args->output_ty = strdup("console");
1332         output_format = mon_output_console;
1333     }
1334 
1335     if (!retval) {
1336         g_propagate_error(&error, err);
1337         clean_up(CRM_EX_USAGE);
1338     }
1339 }
1340 
1341 static void
1342 handle_connection_failures(int rc)
     /*  */
1343 {
1344     if (rc == pcmk_rc_ok) {
1345         return;
1346     }
1347 
1348     if (output_format == mon_output_monitor) {
1349         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "CLUSTER CRIT: Connection to cluster failed: %s",
1350                     pcmk_rc_str(rc));
1351         rc = MON_STATUS_CRIT;
1352     } else if (rc == ENOTCONN) {
1353         if (on_remote_node) {
1354             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: remote-node not connected to cluster");
1355         } else {
1356             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");
1357         }
1358         rc = pcmk_rc2exitc(rc);
1359     } else {
1360         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Connection to cluster failed: %s", pcmk_rc_str(rc));
1361         rc = pcmk_rc2exitc(rc);
1362     }
1363 
1364     clean_up(rc);
1365 }
1366 
1367 static void
1368 one_shot(void)
     /*  */
1369 {
1370     int rc;
1371 
1372     rc = pacemakerd_status();
1373 
1374     if (rc == pcmk_rc_ok) {
1375         fencing_connect();
1376         rc = cib_connect(FALSE);
1377     }
1378 
1379     if (rc == pcmk_rc_ok) {
1380         mon_refresh_display(NULL);
1381     } else {
1382         handle_connection_failures(rc);
1383     }
1384 
1385     clean_up(CRM_EX_OK);
1386 }
1387 
1388 int
1389 main(int argc, char **argv)
     /*  */
1390 {
1391     int rc = pcmk_ok;
1392     GOptionGroup *output_group = NULL;
1393 
1394     args = pcmk__new_common_args(SUMMARY);
1395     context = build_arg_context(args, &output_group);
1396     pcmk__register_formats(output_group, formats);
1397 
1398     options.pid_file = strdup("/tmp/ClusterMon.pid");
1399     pcmk__cli_init_logging("crm_mon", 0);
1400 
1401     // Avoid needing to wait for subprocesses forked for -E/--external-agent
1402     avoid_zombies();
1403 
1404     if (pcmk__ends_with_ext(argv[0], ".cgi")) {
1405         output_format = mon_output_cgi;
1406         options.one_shot = TRUE;
1407     }
1408 
1409     processed_args = pcmk__cmdline_preproc(argv, "ehimpxEILU");
1410 
1411     fence_history_cb("--fence-history", "1", NULL, NULL);
1412 
1413     /* Set an HTML title regardless of what format we will eventually use.  This can't
1414      * be done in add_output_args.  That function is called after command line
1415      * arguments are processed in the next block, which means it'll override whatever
1416      * title the user provides.  Doing this here means the user can give their own
1417      * title on the command line.
1418      */
1419     if (!pcmk__force_args(context, &error, "%s --html-title \"Cluster Status\"",
1420                           g_get_prgname())) {
1421         return clean_up(CRM_EX_USAGE);
1422     }
1423 
1424     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
1425         return clean_up(CRM_EX_USAGE);
1426     }
1427 
1428     for (int i = 0; i < args->verbosity; i++) {
1429         crm_bump_log_level(argc, argv);
1430     }
1431 
1432     if (!args->version) {
1433         if (args->quiet) {
1434             include_exclude_cb("--exclude", "times", NULL, NULL);
1435         }
1436 
1437         if (options.watch_fencing) {
1438             fence_history_cb("--fence-history", "0", NULL, NULL);
1439             options.fence_connect = TRUE;
1440         }
1441 
1442         /* create the cib-object early to be able to do further
1443          * decisions based on the cib-source
1444          */
1445         cib = cib_new();
1446 
1447         if (cib == NULL) {
1448             rc = -EINVAL;
1449         } else {
1450             switch (cib->variant) {
1451 
1452                 case cib_native:
1453                     /* cib & fencing - everything available */
1454                     use_cib_native = TRUE;
1455                     break;
1456 
1457                 case cib_file:
1458                     /* Don't try to connect to fencing as we
1459                      * either don't have a running cluster or
1460                      * the fencing-information would possibly
1461                      * not match the cib data from a file.
1462                      * As we don't expect cib-updates coming
1463                      * in enforce one-shot. */
1464                     fence_history_cb("--fence-history", "0", NULL, NULL);
1465                     options.one_shot = TRUE;
1466                     break;
1467 
1468                 case cib_remote:
1469                     /* updates coming in but no fencing */
1470                     fence_history_cb("--fence-history", "0", NULL, NULL);
1471                     break;
1472 
1473                 case cib_undefined:
1474                 case cib_database:
1475                 default:
1476                     /* something is odd */
1477                     rc = -EINVAL;
1478                     break;
1479             }
1480         }
1481 
1482         if (options.one_shot) {
1483             if (output_format == mon_output_console) {
1484                 output_format = mon_output_plain;
1485             }
1486 
1487         } else if (options.daemonize) {
1488             if ((output_format == mon_output_console) || (args->output_dest == NULL)) {
1489                 output_format = mon_output_none;
1490             }
1491             crm_enable_stderr(FALSE);
1492 
1493             if (pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches | pcmk__str_casei) && !options.external_agent) {
1494                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "--daemonize requires at least one of --output-to and --external-agent");
1495                 return clean_up(CRM_EX_USAGE);
1496             }
1497 
1498             if (cib) {
1499                 /* to be on the safe side don't have cib-object around
1500                  * when we are forking
1501                  */
1502                 cib_delete(cib);
1503                 cib = NULL;
1504                 pcmk__daemonize(crm_system_name, options.pid_file);
1505                 cib = cib_new();
1506                 if (cib == NULL) {
1507                     rc = -EINVAL;
1508                 }
1509                 /* otherwise assume we've got the same cib-object we've just destroyed
1510                  * in our parent
1511                  */
1512             }
1513 
1514 
1515         } else if (output_format == mon_output_console) {
1516 #if CURSES_ENABLED
1517             crm_enable_stderr(FALSE);
1518 #else
1519             options.one_shot = TRUE;
1520             output_format = mon_output_plain;
1521             printf("Defaulting to one-shot mode\n");
1522             printf("You need to have curses available at compile time to enable console mode\n");
1523 #endif
1524         }
1525     }
1526 
1527     if (rc != pcmk_ok) {
1528         // Shouldn't really be possible
1529         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Invalid CIB source");
1530         return clean_up(CRM_EX_ERROR);
1531     }
1532 
1533     reconcile_output_format(args);
1534     add_output_args();
1535 
1536     if (args->version && output_format == mon_output_console) {
1537         /* Use the text output format here if we are in curses mode but were given
1538          * --version.  Displaying version information uses printf, and then we
1539          *  immediately exit.  We don't want to initialize curses for that.
1540          */
1541         rc = pcmk__output_new(&out, "text", args->output_dest, argv);
1542     } else {
1543         rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
1544     }
1545 
1546     if (rc != pcmk_rc_ok) {
1547         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error creating output format %s: %s",
1548                     args->output_ty, pcmk_rc_str(rc));
1549         return clean_up(CRM_EX_ERROR);
1550     }
1551 
1552     /* output_format MUST NOT BE CHANGED AFTER THIS POINT. */
1553 
1554     /* Apply --include/--exclude flags we used internally.  There's no error reporting
1555      * here because this would be a programming error.
1556      */
1557     apply_include_exclude(options.includes_excludes, output_format, &error);
1558 
1559     /* And now apply any --include/--exclude flags the user gave on the command line.
1560      * These are done in a separate pass from the internal ones because we want to
1561      * make sure whatever the user specifies overrides whatever we do.
1562      */
1563     if (!apply_include_exclude(options.user_includes_excludes, output_format, &error)) {
1564         return clean_up(CRM_EX_USAGE);
1565     }
1566 
1567     /* Sync up the initial value of interactive_fence_level with whatever was set with
1568      * --include/--exclude= options.
1569      */
1570     if (pcmk_all_flags_set(show, pcmk_section_fencing_all)) {
1571         interactive_fence_level = 3;
1572     } else if (pcmk_is_set(show, pcmk_section_fence_worked)) {
1573         interactive_fence_level = 2;
1574     } else if (pcmk_any_flags_set(show, pcmk_section_fence_failed | pcmk_section_fence_pending)) {
1575         interactive_fence_level = 1;
1576     } else {
1577         interactive_fence_level = 0;
1578     }
1579 
1580     pcmk__register_lib_messages(out);
1581     crm_mon_register_messages(out);
1582     pe__register_messages(out);
1583     stonith__register_messages(out);
1584 
1585     if (args->version) {
1586         out->version(out, false);
1587         return clean_up(CRM_EX_OK);
1588     }
1589 
1590     /* Extra sanity checks when in CGI mode */
1591     if (output_format == mon_output_cgi) {
1592         if (cib->variant == cib_file) {
1593             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode used with CIB file");
1594             return clean_up(CRM_EX_USAGE);
1595         } else if (options.external_agent != NULL) {
1596             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with --external-agent");
1597             return clean_up(CRM_EX_USAGE);
1598         } else if (options.daemonize == TRUE) {
1599             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with -d");
1600             return clean_up(CRM_EX_USAGE);
1601         }
1602     }
1603 
1604     if (output_format == mon_output_xml || output_format == mon_output_legacy_xml) {
1605         show_opts |= pcmk_show_inactive_rscs | pcmk_show_timing;
1606 
1607         if (!options.daemonize) {
1608             options.one_shot = TRUE;
1609         }
1610     }
1611 
1612     if ((output_format == mon_output_html || output_format == mon_output_cgi) &&
1613         out->dest != stdout) {
1614         pcmk__html_add_header("meta", "http-equiv", "refresh", "content",
1615                               pcmk__itoa(options.reconnect_ms / 1000), NULL);
1616     }
1617 
1618     crm_info("Starting %s", crm_system_name);
1619 
1620     cib__set_output(cib, out);
1621 
1622     if (options.one_shot) {
1623         one_shot();
1624     }
1625 
1626     do {
1627         out->info(out,"Waiting until cluster is available on this node ...");
1628 
1629         rc = pacemakerd_status();
1630         if (rc == pcmk_rc_ok) {
1631             fencing_connect();
1632             rc = cib_connect(TRUE);
1633         }
1634 
1635         if (rc != pcmk_rc_ok) {
1636             pcmk__sleep_ms(options.reconnect_ms);
1637 #if CURSES_ENABLED
1638             if (output_format == mon_output_console) {
1639                 clear();
1640                 refresh();
1641             }
1642 #endif
1643         } else if (output_format == mon_output_html && out->dest != stdout) {
1644             printf("Writing html to %s ...\n", args->output_dest);
1645         }
1646 
1647     } while (rc == ENOTCONN);
1648 
1649     handle_connection_failures(rc);
1650     set_fencing_options(interactive_fence_level);
1651     mon_refresh_display(NULL);
1652 
1653     mainloop = g_main_loop_new(NULL, FALSE);
1654 
1655     mainloop_add_signal(SIGTERM, mon_shutdown);
1656     mainloop_add_signal(SIGINT, mon_shutdown);
1657 #if CURSES_ENABLED
1658     if (output_format == mon_output_console) {
1659         ncurses_winch_handler = crm_signal_handler(SIGWINCH, mon_winresize);
1660         if (ncurses_winch_handler == SIG_DFL ||
1661             ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
1662             ncurses_winch_handler = NULL;
1663 
1664         io_channel = g_io_channel_unix_new(STDIN_FILENO);
1665         g_io_add_watch(io_channel, G_IO_IN, detect_user_input, NULL);
1666     }
1667 #endif
1668 
1669     /* When refresh_trigger->trigger is set to TRUE, call mon_refresh_display.  In
1670      * this file, that is anywhere mainloop_set_trigger is called.
1671      */
1672     refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
1673 
1674     g_main_loop_run(mainloop);
1675     g_main_loop_unref(mainloop);
1676 
1677     if (io_channel != NULL) {
1678         g_io_channel_shutdown(io_channel, TRUE, NULL);
1679     }
1680 
1681     crm_info("Exiting %s", crm_system_name);
1682 
1683     return clean_up(CRM_EX_OK);
1684 }
1685 
1686 /*!
1687  * \internal
1688  * \brief Print one-line status suitable for use with monitoring software
1689  *
1690  * \param[in] data_set  Working set of CIB state
1691  *
1692  * \note This function's output (and the return code when the program exits)
1693  *       should conform to https://www.monitoring-plugins.org/doc/guidelines.html
1694  */
1695 static void
1696 print_simple_status(pcmk__output_t *out, pe_working_set_t * data_set)
     /*  */
1697 {
1698     GList *gIter = NULL;
1699     int nodes_online = 0;
1700     int nodes_standby = 0;
1701     int nodes_maintenance = 0;
1702     char *offline_nodes = NULL;
1703     size_t offline_nodes_len = 0;
1704     gboolean no_dc = FALSE;
1705     gboolean offline = FALSE;
1706 
1707     if (data_set->dc_node == NULL) {
1708         has_warnings = TRUE;
1709         no_dc = TRUE;
1710     }
1711 
1712     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1713         pe_node_t *node = (pe_node_t *) gIter->data;
1714 
1715         if (node->details->standby && node->details->online) {
1716             nodes_standby++;
1717         } else if (node->details->maintenance && node->details->online) {
1718             nodes_maintenance++;
1719         } else if (node->details->online) {
1720             nodes_online++;
1721         } else {
1722             char *s = crm_strdup_printf("offline node: %s", node->details->uname);
1723             /* coverity[leaked_storage] False positive */
1724             pcmk__add_word(&offline_nodes, &offline_nodes_len, s);
1725             free(s);
1726             has_warnings = TRUE;
1727             offline = TRUE;
1728         }
1729     }
1730 
1731     if (has_warnings) {
1732         out->info(out, "CLUSTER WARN: %s%s%s",
1733                   no_dc ? "No DC" : "",
1734                   no_dc && offline ? ", " : "",
1735                   (offline? offline_nodes : ""));
1736         free(offline_nodes);
1737     } else {
1738         char *nodes_standby_s = NULL;
1739         char *nodes_maint_s = NULL;
1740 
1741         if (nodes_standby > 0) {
1742             nodes_standby_s = crm_strdup_printf(", %d standby node%s", nodes_standby,
1743                                                 pcmk__plural_s(nodes_standby));
1744         }
1745 
1746         if (nodes_maintenance > 0) {
1747             nodes_maint_s = crm_strdup_printf(", %d maintenance node%s",
1748                                               nodes_maintenance,
1749                                               pcmk__plural_s(nodes_maintenance));
1750         }
1751 
1752         out->info(out, "CLUSTER OK: %d node%s online%s%s, "
1753                        "%d resource instance%s configured",
1754                   nodes_online, pcmk__plural_s(nodes_online),
1755                   nodes_standby_s != NULL ? nodes_standby_s : "",
1756                   nodes_maint_s != NULL ? nodes_maint_s : "",
1757                   data_set->ninstances, pcmk__plural_s(data_set->ninstances));
1758 
1759         free(nodes_standby_s);
1760         free(nodes_maint_s);
1761     }
1762     /* coverity[leaked_storage] False positive */
1763 }
1764 
1765 static int
1766 send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
     /*  */
1767                  int status, const char *desc)
1768 {
1769     pid_t pid;
1770 
1771     /*setenv needs chars, these are ints */
1772     char *rc_s = pcmk__itoa(rc);
1773     char *status_s = pcmk__itoa(status);
1774     char *target_rc_s = pcmk__itoa(target_rc);
1775 
1776     crm_debug("Sending external notification to '%s' via '%s'", options.external_recipient, options.external_agent);
1777 
1778     if(rsc) {
1779         setenv("CRM_notify_rsc", rsc, 1);
1780     }
1781     if (options.external_recipient) {
1782         setenv("CRM_notify_recipient", options.external_recipient, 1);
1783     }
1784     setenv("CRM_notify_node", node, 1);
1785     setenv("CRM_notify_task", task, 1);
1786     setenv("CRM_notify_desc", desc, 1);
1787     setenv("CRM_notify_rc", rc_s, 1);
1788     setenv("CRM_notify_target_rc", target_rc_s, 1);
1789     setenv("CRM_notify_status", status_s, 1);
1790 
1791     pid = fork();
1792     if (pid == -1) {
1793         crm_perror(LOG_ERR, "notification fork() failed.");
1794     }
1795     if (pid == 0) {
1796         /* crm_debug("notification: I am the child. Executing the nofitication program."); */
1797         execl(options.external_agent, options.external_agent, NULL);
1798         exit(CRM_EX_ERROR);
1799     }
1800 
1801     crm_trace("Finished running custom notification program '%s'.", options.external_agent);
1802     free(target_rc_s);
1803     free(status_s);
1804     free(rc_s);
1805     return 0;
1806 }
1807 
1808 static void
1809 handle_rsc_op(xmlNode * xml, const char *node_id)
     /*  */
1810 {
1811     int rc = -1;
1812     int status = -1;
1813     int target_rc = -1;
1814     gboolean notify = TRUE;
1815 
1816     char *rsc = NULL;
1817     char *task = NULL;
1818     const char *desc = NULL;
1819     const char *magic = NULL;
1820     const char *id = NULL;
1821     const char *node = NULL;
1822 
1823     xmlNode *n = xml;
1824     xmlNode * rsc_op = xml;
1825 
1826     if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
1827         xmlNode *cIter;
1828 
1829         for(cIter = xml->children; cIter; cIter = cIter->next) {
1830             handle_rsc_op(cIter, node_id);
1831         }
1832 
1833         return;
1834     }
1835 
1836     id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
1837     if (id == NULL) {
1838         /* Compatibility with <= 1.1.5 */
1839         id = ID(rsc_op);
1840     }
1841 
1842     magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
1843     if (magic == NULL) {
1844         /* non-change */
1845         return;
1846     }
1847 
1848     if (!decode_transition_magic(magic, NULL, NULL, NULL, &status, &rc,
1849                                  &target_rc)) {
1850         crm_err("Invalid event %s detected for %s", magic, id);
1851         return;
1852     }
1853 
1854     if (parse_op_key(id, &rsc, &task, NULL) == FALSE) {
1855         crm_err("Invalid event detected for %s", id);
1856         goto bail;
1857     }
1858 
1859     node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
1860 
1861     while (n != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(n), pcmk__str_casei)) {
1862         n = n->parent;
1863     }
1864 
1865     if(node == NULL && n) {
1866         node = crm_element_value(n, XML_ATTR_UNAME);
1867     }
1868 
1869     if (node == NULL && n) {
1870         node = ID(n);
1871     }
1872 
1873     if (node == NULL) {
1874         node = node_id;
1875     }
1876 
1877     if (node == NULL) {
1878         crm_err("No node detected for event %s (%s)", magic, id);
1879         goto bail;
1880     }
1881 
1882     /* look up where we expected it to be? */
1883     desc = pcmk_strerror(pcmk_ok);
1884     if (status == PCMK_LRM_OP_DONE && target_rc == rc) {
1885         crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
1886         if (rc == PCMK_OCF_NOT_RUNNING) {
1887             notify = FALSE;
1888         }
1889 
1890     } else if (status == PCMK_LRM_OP_DONE) {
1891         desc = services_ocf_exitcode_str(rc);
1892         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1893 
1894     } else {
1895         desc = services_lrm_status_str(status);
1896         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1897     }
1898 
1899     if (notify && options.external_agent) {
1900         send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
1901     }
1902   bail:
1903     free(rsc);
1904     free(task);
1905 }
1906 
1907 /* This function is just a wrapper around mainloop_set_trigger so that it can be
1908  * called from a mainloop directly.  It's simply another way of ensuring the screen
1909  * gets redrawn.
1910  */
1911 static gboolean
1912 mon_trigger_refresh(gpointer user_data)
     /*  */
1913 {
1914     mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
1915     return FALSE;
1916 }
1917 
1918 static void
1919 crm_diff_update_v2(const char *event, xmlNode * msg)
     /*  */
1920 {
1921     xmlNode *change = NULL;
1922     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
1923 
1924     for (change = pcmk__xml_first_child(diff); change != NULL;
1925          change = pcmk__xml_next(change)) {
1926         const char *name = NULL;
1927         const char *op = crm_element_value(change, XML_DIFF_OP);
1928         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
1929         xmlNode *match = NULL;
1930         const char *node = NULL;
1931 
1932         if(op == NULL) {
1933             continue;
1934 
1935         } else if(strcmp(op, "create") == 0) {
1936             match = change->children;
1937 
1938         } else if(strcmp(op, "move") == 0) {
1939             continue;
1940 
1941         } else if(strcmp(op, "delete") == 0) {
1942             continue;
1943 
1944         } else if(strcmp(op, "modify") == 0) {
1945             match = first_named_child(change, XML_DIFF_RESULT);
1946             if(match) {
1947                 match = match->children;
1948             }
1949         }
1950 
1951         if(match) {
1952             name = (const char *)match->name;
1953         }
1954 
1955         crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
1956         if(xpath == NULL) {
1957             /* Version field, ignore */
1958 
1959         } else if(name == NULL) {
1960             crm_debug("No result for %s operation to %s", op, xpath);
1961             CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
1962 
1963         } else if(strcmp(name, XML_TAG_CIB) == 0) {
1964             xmlNode *state = NULL;
1965             xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
1966 
1967             for (state = pcmk__xe_first_child(status); state != NULL;
1968                  state = pcmk__xe_next(state)) {
1969 
1970                 node = crm_element_value(state, XML_ATTR_UNAME);
1971                 if (node == NULL) {
1972                     node = ID(state);
1973                 }
1974                 handle_rsc_op(state, node);
1975             }
1976 
1977         } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
1978             xmlNode *state = NULL;
1979 
1980             for (state = pcmk__xe_first_child(match); state != NULL;
1981                  state = pcmk__xe_next(state)) {
1982 
1983                 node = crm_element_value(state, XML_ATTR_UNAME);
1984                 if (node == NULL) {
1985                     node = ID(state);
1986                 }
1987                 handle_rsc_op(state, node);
1988             }
1989 
1990         } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
1991             node = crm_element_value(match, XML_ATTR_UNAME);
1992             if (node == NULL) {
1993                 node = ID(match);
1994             }
1995             handle_rsc_op(match, node);
1996 
1997         } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
1998             node = ID(match);
1999             handle_rsc_op(match, node);
2000 
2001         } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
2002             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2003 
2004             handle_rsc_op(match, local_node);
2005             free(local_node);
2006 
2007         } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
2008             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2009 
2010             handle_rsc_op(match, local_node);
2011             free(local_node);
2012 
2013         } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
2014             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2015 
2016             handle_rsc_op(match, local_node);
2017             free(local_node);
2018 
2019         } else {
2020             crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
2021         }
2022     }
2023 }
2024 
2025 static void
2026 crm_diff_update_v1(const char *event, xmlNode * msg)
     /*  */
2027 {
2028     /* Process operation updates */
2029     xmlXPathObject *xpathObj = xpath_search(msg,
2030                                             "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
2031                                             "//" XML_LRM_TAG_RSC_OP);
2032     int lpc = 0, max = numXpathResults(xpathObj);
2033 
2034     for (lpc = 0; lpc < max; lpc++) {
2035         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
2036 
2037         handle_rsc_op(rsc_op, NULL);
2038     }
2039     freeXpathObject(xpathObj);
2040 }
2041 
2042 static void
2043 crm_diff_update(const char *event, xmlNode * msg)
     /*  */
2044 {
2045     int rc = -1;
2046     static bool stale = FALSE;
2047     gboolean cib_updated = FALSE;
2048     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
2049 
2050     out->progress(out, false);
2051 
2052     if (current_cib != NULL) {
2053         rc = xml_apply_patchset(current_cib, diff, TRUE);
2054 
2055         switch (rc) {
2056             case -pcmk_err_diff_resync:
2057             case -pcmk_err_diff_failed:
2058                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
2059                 free_xml(current_cib); current_cib = NULL;
2060                 break;
2061             case pcmk_ok:
2062                 cib_updated = TRUE;
2063                 break;
2064             default:
2065                 crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
2066                 free_xml(current_cib); current_cib = NULL;
2067         }
2068     }
2069 
2070     if (current_cib == NULL) {
2071         crm_trace("Re-requesting the full cib");
2072         cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
2073     }
2074 
2075     if (options.external_agent) {
2076         int format = 0;
2077         crm_element_value_int(diff, "format", &format);
2078         switch(format) {
2079             case 1:
2080                 crm_diff_update_v1(event, msg);
2081                 break;
2082             case 2:
2083                 crm_diff_update_v2(event, msg);
2084                 break;
2085             default:
2086                 crm_err("Unknown patch format: %d", format);
2087         }
2088     }
2089 
2090     if (current_cib == NULL) {
2091         if(!stale) {
2092             out->info(out, "--- Stale data ---");
2093         }
2094         stale = TRUE;
2095         return;
2096     }
2097 
2098     stale = FALSE;
2099     refresh_after_event(cib_updated, FALSE);
2100 }
2101 
2102 static int
2103 get_fencing_history(stonith_history_t **stonith_history)
     /*  */
2104 {
2105     int rc = 0;
2106 
2107     while (fence_history) {
2108         if (st != NULL) {
2109             rc = st->cmds->history(st, st_opt_sync_call, NULL, stonith_history, 120);
2110 
2111             if (rc == 0) {
2112                 *stonith_history = stonith__sort_history(*stonith_history);
2113                 if (!pcmk_all_flags_set(show, pcmk_section_fencing_all)
2114                     && (output_format != mon_output_xml)) {
2115 
2116                     *stonith_history = pcmk__reduce_fence_history(*stonith_history);
2117                 }
2118                 break; /* all other cases are errors */
2119             }
2120         } else {
2121             rc = ENOTCONN;
2122             break;
2123         }
2124     }
2125 
2126     return rc;
2127 }
2128 
2129 static int
2130 mon_refresh_display(gpointer user_data)
     /*  */
2131 {
2132     xmlNode *cib_copy = copy_xml(current_cib);
2133     stonith_history_t *stonith_history = NULL;
2134     int history_rc = 0;
2135     GList *unames = NULL;
2136     GList *resources = NULL;
2137 
2138     last_refresh = time(NULL);
2139 
2140     if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
2141         clean_up_cib_connection();
2142         out->err(out, "Upgrade failed: %s", pcmk_strerror(-pcmk_err_schema_validation));
2143         clean_up(CRM_EX_CONFIG);
2144         return 0;
2145     }
2146 
2147     /* get the stonith-history if there is evidence we need it */
2148     history_rc = get_fencing_history(&stonith_history);
2149 
2150     if (mon_data_set == NULL) {
2151         mon_data_set = pe_new_working_set();
2152         CRM_ASSERT(mon_data_set != NULL);
2153     }
2154     pe__set_working_set_flags(mon_data_set, pe_flag_no_compat);
2155 
2156     mon_data_set->input = cib_copy;
2157     mon_data_set->priv = out;
2158     cluster_status(mon_data_set);
2159 
2160     /* Unpack constraints if any section will need them
2161      * (tickets may be referenced in constraints but not granted yet,
2162      * and bans need negative location constraints) */
2163     if (pcmk_is_set(show, pcmk_section_bans) || pcmk_is_set(show, pcmk_section_tickets)) {
2164         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
2165                                                    mon_data_set->input);
2166         unpack_constraints(cib_constraints, mon_data_set);
2167     }
2168 
2169     if (options.daemonize) {
2170         out->reset(out);
2171     }
2172 
2173     unames = pe__build_node_name_list(mon_data_set, options.only_node);
2174     resources = pe__build_rsc_list(mon_data_set, options.only_rsc);
2175 
2176     switch (output_format) {
2177         case mon_output_html:
2178         case mon_output_cgi:
2179             if (out->message(out, "cluster-status", mon_data_set, crm_errno2exit(history_rc),
2180                              stonith_history, fence_history, show, show_opts,
2181                              options.neg_location_prefix, unames, resources) != 0) {
2182                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_CANTCREAT, "Critical: Unable to output html file");
2183                 clean_up(CRM_EX_CANTCREAT);
2184                 return 0;
2185             }
2186             break;
2187 
2188         case mon_output_monitor:
2189             print_simple_status(out, mon_data_set);
2190             if (has_warnings) {
2191                 clean_up(MON_STATUS_WARN);
2192                 return FALSE;
2193             }
2194             break;
2195 
2196         case mon_output_unset:
2197         case mon_output_none:
2198             break;
2199 
2200         default:
2201             out->message(out, "cluster-status", mon_data_set, crm_errno2exit(history_rc),
2202                          stonith_history, fence_history, show, show_opts,
2203                          options.neg_location_prefix, unames, resources);
2204             break;
2205     }
2206 
2207     if (options.daemonize) {
2208         out->finish(out, CRM_EX_OK, true, NULL);
2209     }
2210 
2211     g_list_free_full(unames, free);
2212     g_list_free_full(resources, free);
2213 
2214     stonith_history_free(stonith_history);
2215     stonith_history = NULL;
2216     pe_reset_working_set(mon_data_set);
2217     return 1;
2218 }
2219 
2220 /* This function is called for fencing events (see fencing_connect for which ones) when
2221  * --watch-fencing is used on the command line.
2222  */
2223 static void
2224 mon_st_callback_event(stonith_t * st, stonith_event_t * e)
     /*  */
2225 {
2226     if (st->state == stonith_disconnected) {
2227         /* disconnect cib as well and have everything reconnect */
2228         mon_cib_connection_destroy(NULL);
2229     } else if (options.external_agent) {
2230         char *desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
2231                                     e->operation, e->origin, e->target, pcmk_strerror(e->result),
2232                                     e->id);
2233         send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
2234         free(desc);
2235     }
2236 }
2237 
2238 /* Cause the screen to be redrawn (via mainloop_set_trigger) when various conditions are met:
2239  *
2240  * - If the last update occurred more than reconnect_ms ago (defaults to 5s, but
2241  *   can be changed via the -i command line option), or
2242  * - After every 10 CIB updates, or
2243  * - If it's been 2s since the last update
2244  *
2245  * This function sounds like it would be more broadly useful, but it is only called when a
2246  * fencing event is received or a CIB diff occurrs.
2247  */
2248 static void
2249 refresh_after_event(gboolean data_updated, gboolean enforce)
     /*  */
2250 {
2251     static int updates = 0;
2252     time_t now = time(NULL);
2253 
2254     if (data_updated) {
2255         updates++;
2256     }
2257 
2258     if(refresh_timer == NULL) {
2259         refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
2260     }
2261 
2262     if (reconnect_timer > 0) {
2263         /* we will receive a refresh request after successful reconnect */
2264         mainloop_timer_stop(refresh_timer);
2265         return;
2266     }
2267 
2268     /* as we're not handling initial failure of fencer-connection as
2269      * fatal give it a retry here
2270      * not getting here if cib-reconnection is already on the way
2271      */
2272     fencing_connect();
2273 
2274     if (enforce ||
2275         ((now - last_refresh) > (options.reconnect_ms / 1000)) ||
2276         updates >= 10) {
2277         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
2278         mainloop_timer_stop(refresh_timer);
2279         updates = 0;
2280 
2281     } else {
2282         mainloop_timer_start(refresh_timer);
2283     }
2284 }
2285 
2286 /* This function is called for fencing events (see fencing_connect for which ones) when
2287  * --watch-fencing is NOT used on the command line.
2288  */
2289 static void
2290 mon_st_callback_display(stonith_t * st, stonith_event_t * e)
     /*  */
2291 {
2292     if (st->state == stonith_disconnected) {
2293         /* disconnect cib as well and have everything reconnect */
2294         mon_cib_connection_destroy(NULL);
2295     } else {
2296         out->progress(out, false);
2297         refresh_after_event(TRUE, FALSE);
2298     }
2299 }
2300 
2301 static void
2302 clean_up_cib_connection(void)
     /*  */
2303 {
2304     if (cib == NULL) {
2305         return;
2306     }
2307 
2308     cib->cmds->signoff(cib);
2309     cib_delete(cib);
2310     cib = NULL;
2311 }
2312 
2313 static void
2314 clean_up_fencing_connection(void)
     /*  */
2315 {
2316     if (st == NULL) {
2317         return;
2318     }
2319 
2320     if (st->state != stonith_disconnected) {
2321         st->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT);
2322         st->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE);
2323         st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY);
2324         st->cmds->disconnect(st);
2325     }
2326 
2327     stonith_api_delete(st);
2328     st = NULL;
2329 }
2330 
2331 /*
2332  * De-init ncurses, disconnect from the CIB manager, disconnect fencing,
2333  * deallocate memory and show usage-message if requested.
2334  *
2335  * We don't actually return, but nominally returning crm_exit_t allows a usage
2336  * like "return clean_up(exit_code);" which helps static analysis understand the
2337  * code flow.
2338  */
2339 static crm_exit_t
2340 clean_up(crm_exit_t exit_code)
     /*  */
2341 {
2342     /* Quitting crm_mon is much more complicated than it ought to be. */
2343 
2344     /* (1) Close connections, free things, etc. */
2345     clean_up_cib_connection();
2346     clean_up_fencing_connection();
2347     free(options.neg_location_prefix);
2348     free(options.only_node);
2349     free(options.only_rsc);
2350     free(options.pid_file);
2351     g_slist_free_full(options.includes_excludes, free);
2352 
2353     pe_free_working_set(mon_data_set);
2354     mon_data_set = NULL;
2355 
2356     g_strfreev(processed_args);
2357 
2358     /* (2) If this is abnormal termination and we're in curses mode, shut down
2359      * curses first.  Any messages displayed to the screen before curses is shut
2360      * down will be lost because doing the shut down will also restore the
2361      * screen to whatever it looked like before crm_mon was started.
2362      */
2363     if ((error != NULL || exit_code == CRM_EX_USAGE) && output_format == mon_output_console) {
2364         out->finish(out, exit_code, false, NULL);
2365         pcmk__output_free(out);
2366         out = NULL;
2367     }
2368 
2369     /* (3) If this is a command line usage related failure, print the usage
2370      * message.
2371      */
2372     if (exit_code == CRM_EX_USAGE && (output_format == mon_output_console || output_format == mon_output_plain)) {
2373         char *help = g_option_context_get_help(context, TRUE, NULL);
2374 
2375         fprintf(stderr, "%s", help);
2376         g_free(help);
2377     }
2378 
2379     pcmk__free_arg_context(context);
2380 
2381     /* (4) If this is any kind of error, print the error out and exit.  Make
2382      * sure to handle situations both before and after formatted output is
2383      * set up.  We want errors to appear formatted if at all possible.
2384      */
2385     if (error != NULL) {
2386         if (out != NULL) {
2387             out->err(out, "%s: %s", g_get_prgname(), error->message);
2388             out->finish(out, exit_code, true, NULL);
2389             pcmk__output_free(out);
2390         } else {
2391             fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
2392         }
2393 
2394         g_clear_error(&error);
2395         crm_exit(exit_code);
2396     }
2397 
2398     /* (5) Print formatted output to the screen if we made it far enough in
2399      * crm_mon to be able to do so.
2400      */
2401     if (out != NULL) {
2402         if (!options.daemonize) {
2403             out->finish(out, exit_code, true, NULL);
2404         }
2405 
2406         pcmk__output_free(out);
2407         pcmk__unregister_formats();
2408     }
2409 
2410     crm_exit(exit_code);
2411 }