root/tools/crm_mon.c

/* */

DEFINITIONS

1. all_includes
2. default_includes
3. find_section_bit
4. apply_exclude
5. apply_include
6. apply_include_exclude
7. user_include_exclude_cb
8. include_exclude_cb
9. as_cgi_cb
10. as_html_cb
11. as_simple_cb
12. as_xml_cb
13. fence_history_cb
14. group_by_node_cb
15. hide_headers_cb
16. inactive_resources_cb
17. no_curses_cb
18. one_shot_cb
19. print_brief_cb
20. print_clone_detail_cb
21. print_pending_cb
22. print_timing_cb
23. reconnect_cb
24. show_attributes_cb
25. show_bans_cb
26. show_failcounts_cb
27. show_operations_cb
28. show_tickets_cb
29. use_cib_file_cb
30. watch_fencing_cb
31. get_resource_display_options
32. blank_screen
33. reconnect_after_timeout
34. mon_cib_connection_destroy
35. mon_shutdown
36. mon_winresize
37. fencing_connect
38. cib_connect
39. set_fencing_options
40. pacemakerd_event_cb
41. pacemakerd_status
42. get_option_desc
43. detect_user_input
44. avoid_zombies
45. build_arg_context
46. add_output_args
47. reconcile_output_format
48. handle_connection_failures
49. one_shot
50. main
51. print_simple_status
52. send_custom_trap
53. handle_rsc_op
54. mon_trigger_refresh
55. crm_diff_update_v2
56. crm_diff_update_v1
57. crm_diff_update
58. get_fencing_history
59. mon_refresh_display
60. mon_st_callback_event
61. refresh_after_event
62. mon_st_callback_display
63. clean_up_cib_connection
64. clean_up_fencing_connection
65. clean_up

   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 
  14 #include <crm/crm.h>
  15 
  16 #include <stdio.h>
  17 #include <sys/types.h>
  18 #include <sys/stat.h>
  19 #include <unistd.h>
  20 
  21 #include <stdlib.h>
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <libgen.h>
  25 #include <signal.h>
  26 #include <sys/utsname.h>
  27 
  28 #include <crm/msg_xml.h>
  29 #include <crm/services.h>
  30 #include <crm/lrmd.h>
  31 #include <crm/common/cmdline_internal.h>
  32 #include <crm/common/curses_internal.h>
  33 #include <crm/common/internal.h>  // pcmk__ends_with_ext()
  34 #include <crm/common/ipc.h>
  35 #include <crm/common/iso8601_internal.h>
  36 #include <crm/common/mainloop.h>
  37 #include <crm/common/output_internal.h>
  38 #include <crm/common/util.h>
  39 #include <crm/common/xml.h>
  40 #include <crm/common/xml_internal.h>
  41 
  42 #include <crm/cib/internal.h>
  43 #include <crm/pengine/status.h>
  44 #include <crm/pengine/internal.h>
  45 #include <pacemaker-internal.h>
  46 #include <crm/stonith-ng.h>
  47 #include <crm/fencing/internal.h>
  48 
  49 #include "crm_mon.h"
  50 
  51 #define SUMMARY "Provides a summary of cluster's current state.\n\n" \
  52                 "Outputs varying levels of detail in a number of different formats."
  53 
  54 /*
  55  * Definitions indicating which items to print
  56  */
  57 
  58 static unsigned int show;
  59 
  60 /*
  61  * Definitions indicating how to output
  62  */
  63 
  64 static mon_output_format_t output_format = mon_output_unset;
  65 
  66 /* other globals */
  67 static GIOChannel *io_channel = NULL;
  68 static GMainLoop *mainloop = NULL;
  69 static guint reconnect_timer = 0;
  70 static mainloop_timer_t *refresh_timer = NULL;
  71 static pe_working_set_t *mon_data_set = NULL;
  72 
  73 static cib_t *cib = NULL;
  74 static stonith_t *st = NULL;
  75 static xmlNode *current_cib = NULL;
  76 
  77 static GError *error = NULL;
  78 static pcmk__common_args_t *args = NULL;
  79 static pcmk__output_t *out = NULL;
  80 static GOptionContext *context = NULL;
  81 static gchar **processed_args = NULL;
  82 
  83 static time_t last_refresh = 0;
  84 volatile crm_trigger_t *refresh_trigger = NULL;
  85 
  86 static gboolean on_remote_node = FALSE;
  87 
  88 int interactive_fence_level = 0;
  89 
  90 static pcmk__supported_format_t formats[] = {
  91 #if CURSES_ENABLED
  92     CRM_MON_SUPPORTED_FORMAT_CURSES,
  93 #endif
  94     PCMK__SUPPORTED_FORMAT_HTML,
  95     PCMK__SUPPORTED_FORMAT_NONE,
  96     PCMK__SUPPORTED_FORMAT_TEXT,
  97     PCMK__SUPPORTED_FORMAT_XML,
  98     { NULL, NULL, NULL }
  99 };
 100 
 101 /* Define exit codes for monitoring-compatible output
 102  * For nagios plugins, the possibilities are
 103  * OK=0, WARN=1, CRIT=2, and UNKNOWN=3
 104  */
 105 #define MON_STATUS_WARN    CRM_EX_ERROR
 106 #define MON_STATUS_CRIT    CRM_EX_INVALID_PARAM
 107 #define MON_STATUS_UNKNOWN CRM_EX_UNIMPLEMENT_FEATURE
 108 
 109 #define RECONNECT_MSECS 5000
 110 
 111 struct {
 112     guint reconnect_ms;
 113     gboolean daemonize;
 114     gboolean show_bans;
 115     char *pid_file;
 116     char *external_agent;
 117     char *external_recipient;
 118     char *neg_location_prefix;
 119     char *only_node;
 120     char *only_rsc;
 121     unsigned int mon_ops;
 122     GSList *user_includes_excludes;
 123     GSList *includes_excludes;
 124 } options = {
 125     .reconnect_ms = RECONNECT_MSECS,
 126     .mon_ops = mon_op_default
 127 };
 128 
 129 static void clean_up_cib_connection(void);
 130 static void clean_up_fencing_connection(void);
 131 static crm_exit_t clean_up(crm_exit_t exit_code);
 132 static void crm_diff_update(const char *event, xmlNode * msg);
 133 static void handle_connection_failures(int rc);
 134 static int mon_refresh_display(gpointer user_data);
 135 static int cib_connect(gboolean full);
 136 static int fencing_connect(void);
 137 static int pacemakerd_status(void);
 138 static void mon_st_callback_event(stonith_t * st, stonith_event_t * e);
 139 static void mon_st_callback_display(stonith_t * st, stonith_event_t * e);
 140 static void refresh_after_event(gboolean data_updated, gboolean enforce);
 141 
 142 static unsigned int
 143 all_includes(mon_output_format_t fmt) {
     /*  */
 144     if (fmt == mon_output_monitor || fmt == mon_output_plain || fmt == mon_output_console) {
 145         return ~mon_show_options;
 146     } else {
 147         return mon_show_all;
 148     }
 149 }
 150 
 151 static unsigned int
 152 default_includes(mon_output_format_t fmt) {
     /*  */
 153     switch (fmt) {
 154         case mon_output_monitor:
 155         case mon_output_plain:
 156         case mon_output_console:
 157             return mon_show_stack | mon_show_dc | mon_show_times | mon_show_counts |
 158                    mon_show_nodes | mon_show_resources | mon_show_failures;
 159 
 160         case mon_output_xml:
 161         case mon_output_legacy_xml:
 162             return all_includes(fmt);
 163 
 164         case mon_output_html:
 165         case mon_output_cgi:
 166             return mon_show_summary | mon_show_nodes | mon_show_resources |
 167                    mon_show_failures;
 168 
 169         default:
 170             return 0;
 171     }
 172 }
 173 
 174 struct {
 175     const char *name;
 176     unsigned int bit;
 177 } sections[] = {
 178     { "attributes", mon_show_attributes },
 179     { "bans", mon_show_bans },
 180     { "counts", mon_show_counts },
 181     { "dc", mon_show_dc },
 182     { "failcounts", mon_show_failcounts },
 183     { "failures", mon_show_failures },
 184     { "fencing", mon_show_fencing_all },
 185     { "fencing-failed", mon_show_fence_failed },
 186     { "fencing-pending", mon_show_fence_pending },
 187     { "fencing-succeeded", mon_show_fence_worked },
 188     { "nodes", mon_show_nodes },
 189     { "operations", mon_show_operations },
 190     { "options", mon_show_options },
 191     { "resources", mon_show_resources },
 192     { "stack", mon_show_stack },
 193     { "summary", mon_show_summary },
 194     { "tickets", mon_show_tickets },
 195     { "times", mon_show_times },
 196     { NULL }
 197 };
 198 
 199 static unsigned int
 200 find_section_bit(const char *name) {
     /*  */
 201     for (int i = 0; sections[i].name != NULL; i++) {
 202         if (pcmk__str_eq(sections[i].name, name, pcmk__str_casei)) {
 203             return sections[i].bit;
 204         }
 205     }
 206 
 207     return 0;
 208 }
 209 
 210 static gboolean
 211 apply_exclude(const gchar *excludes, GError **error) {
     /*  */
 212     char **parts = NULL;
 213     gboolean result = TRUE;
 214 
 215     parts = g_strsplit(excludes, ",", 0);
 216     for (char **s = parts; *s != NULL; s++) {
 217         unsigned int bit = find_section_bit(*s);
 218 
 219         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 220             show = 0;
 221         } else if (pcmk__str_eq(*s, "none", pcmk__str_none)) {
 222             show = all_includes(output_format);
 223         } else if (bit != 0) {
 224             show &= ~bit;
 225         } else {
 226             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 227                         "--exclude options: all, attributes, bans, counts, dc, "
 228                         "failcounts, failures, fencing, fencing-failed, "
 229                         "fencing-pending, fencing-succeeded, nodes, none, "
 230                         "operations, options, resources, stack, summary, "
 231                         "tickets, times");
 232             result = FALSE;
 233             break;
 234         }
 235     }
 236     g_strfreev(parts);
 237     return result;
 238 }
 239 
 240 static gboolean
 241 apply_include(const gchar *includes, GError **error) {
     /*  */
 242     char **parts = NULL;
 243     gboolean result = TRUE;
 244 
 245     parts = g_strsplit(includes, ",", 0);
 246     for (char **s = parts; *s != NULL; s++) {
 247         unsigned int bit = find_section_bit(*s);
 248 
 249         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 250             show = all_includes(output_format);
 251         } else if (pcmk__starts_with(*s, "bans")) {
 252             show |= mon_show_bans;
 253             if (options.neg_location_prefix != NULL) {
 254                 free(options.neg_location_prefix);
 255                 options.neg_location_prefix = NULL;
 256             }
 257 
 258             if (strlen(*s) > 4 && (*s)[4] == ':') {
 259                 options.neg_location_prefix = strdup(*s+5);
 260             }
 261         } else if (pcmk__str_any_of(*s, "default", "defaults", NULL)) {
 262             show |= default_includes(output_format);
 263         } else if (pcmk__str_eq(*s, "none", pcmk__str_none)) {
 264             show = 0;
 265         } else if (bit != 0) {
 266             show |= bit;
 267         } else {
 268             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 269                         "--include options: all, attributes, bans[:PREFIX], counts, dc, "
 270                         "default, failcounts, failures, fencing, fencing-failed, "
 271                         "fencing-pending, fencing-succeeded, nodes, none, operations, "
 272                         "options, resources, stack, summary, tickets, times");
 273             result = FALSE;
 274             break;
 275         }
 276     }
 277     g_strfreev(parts);
 278     return result;
 279 }
 280 
 281 static gboolean
 282 apply_include_exclude(GSList *lst, mon_output_format_t fmt, GError **error) {
     /*  */
 283     gboolean rc = TRUE;
 284     GSList *node = lst;
 285 
 286     /* Set the default of what to display here.  Note that we OR everything to
 287      * show instead of set show directly because it could have already had some
 288      * settings applied to it in main.
 289      */
 290     show |= default_includes(fmt);
 291 
 292     while (node != NULL) {
 293         char *s = node->data;
 294 
 295         if (pcmk__starts_with(s, "--include=")) {
 296             rc = apply_include(s+10, error);
 297         } else if (pcmk__starts_with(s, "-I=")) {
 298             rc = apply_include(s+3, error);
 299         } else if (pcmk__starts_with(s, "--exclude=")) {
 300             rc = apply_exclude(s+10, error);
 301         } else if (pcmk__starts_with(s, "-U=")) {
 302             rc = apply_exclude(s+3, error);
 303         }
 304 
 305         if (rc != TRUE) {
 306             break;
 307         }
 308 
 309         node = node->next;
 310     }
 311 
 312     return rc;
 313 }
 314 
 315 static gboolean
 316 user_include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 317     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 318 
 319     options.user_includes_excludes = g_slist_append(options.user_includes_excludes, s);
 320     return TRUE;
 321 }
 322 
 323 static gboolean
 324 include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 325     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 326 
 327     options.includes_excludes = g_slist_append(options.includes_excludes, s);
 328     return TRUE;
 329 }
 330 
 331 static gboolean
 332 as_cgi_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 333     if (args->output_ty != NULL) {
 334         free(args->output_ty);
 335     }
 336 
 337     args->output_ty = strdup("html");
 338     output_format = mon_output_cgi;
 339     options.mon_ops |= mon_op_one_shot;
 340     return TRUE;
 341 }
 342 
 343 static gboolean
 344 as_html_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 345     if (args->output_ty != NULL) {
 346         free(args->output_ty);
 347     }
 348 
 349     if (args->output_dest != NULL) {
 350         free(args->output_dest);
 351         args->output_dest = NULL;
 352     }
 353 
 354     if (optarg != NULL) {
 355         args->output_dest = strdup(optarg);
 356     }
 357 
 358     args->output_ty = strdup("html");
 359     output_format = mon_output_html;
 360     umask(S_IWGRP | S_IWOTH);
 361     return TRUE;
 362 }
 363 
 364 static gboolean
 365 as_simple_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 366     if (args->output_ty != NULL) {
 367         free(args->output_ty);
 368     }
 369 
 370     args->output_ty = strdup("text");
 371     output_format = mon_output_monitor;
 372     options.mon_ops |= mon_op_one_shot;
 373     return TRUE;
 374 }
 375 
 376 static gboolean
 377 as_xml_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 378     if (args->output_ty != NULL) {
 379         free(args->output_ty);
 380     }
 381 
 382     args->output_ty = strdup("xml");
 383     output_format = mon_output_legacy_xml;
 384     return TRUE;
 385 }
 386 
 387 static gboolean
 388 fence_history_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 389     if (optarg == NULL) {
 390         interactive_fence_level = 2;
 391     } else {
 392         pcmk__scan_min_int(optarg, &interactive_fence_level, 0);
 393     }
 394 
 395     switch (interactive_fence_level) {
 396         case 3:
 397             options.mon_ops |= mon_op_fence_full_history | mon_op_fence_history | mon_op_fence_connect;
 398             return include_exclude_cb("--include", "fencing", data, err);
 399 
 400         case 2:
 401             options.mon_ops |= mon_op_fence_history | mon_op_fence_connect;
 402             return include_exclude_cb("--include", "fencing", data, err);
 403 
 404         case 1:
 405             options.mon_ops |= mon_op_fence_history | mon_op_fence_connect;
 406             return include_exclude_cb("--include", "fencing-failed,fencing-pending", data, err);
 407 
 408         case 0:
 409             options.mon_ops &= ~(mon_op_fence_history | mon_op_fence_connect);
 410             return include_exclude_cb("--exclude", "fencing", data, err);
 411 
 412         default:
 413             g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Fence history must be 0-3");
 414             return FALSE;
 415     }
 416 }
 417 
 418 static gboolean
 419 group_by_node_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 420     options.mon_ops |= mon_op_group_by_node;
 421     return TRUE;
 422 }
 423 
 424 static gboolean
 425 hide_headers_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 426     return include_exclude_cb("--exclude", "summary", data, err);
 427 }
 428 
 429 static gboolean
 430 inactive_resources_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 431     options.mon_ops |= mon_op_inactive_resources;
 432     return TRUE;
 433 }
 434 
 435 static gboolean
 436 no_curses_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 437     output_format = mon_output_plain;
 438     return TRUE;
 439 }
 440 
 441 static gboolean
 442 one_shot_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 443     options.mon_ops |= mon_op_one_shot;
 444     return TRUE;
 445 }
 446 
 447 static gboolean
 448 print_brief_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 449     options.mon_ops |= mon_op_print_brief;
 450     return TRUE;
 451 }
 452 
 453 static gboolean
 454 print_clone_detail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 455     options.mon_ops |= mon_op_print_clone_detail;
 456     return TRUE;
 457 }
 458 
 459 static gboolean
 460 print_pending_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 461     options.mon_ops |= mon_op_print_pending;
 462     return TRUE;
 463 }
 464 
 465 static gboolean
 466 print_timing_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 467     options.mon_ops |= mon_op_print_timing;
 468     return include_exclude_cb("--include", "operations", data, err);
 469 }
 470 
 471 static gboolean
 472 reconnect_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 473     int rc = crm_get_msec(optarg);
 474 
 475     if (rc == -1) {
 476         g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid value for -i: %s", optarg);
 477         return FALSE;
 478     } else {
 479         options.reconnect_ms = crm_parse_interval_spec(optarg);
 480     }
 481 
 482     return TRUE;
 483 }
 484 
 485 static gboolean
 486 show_attributes_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 487     return include_exclude_cb("--include", "attributes", data, err);
 488 }
 489 
 490 static gboolean
 491 show_bans_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 492     if (optarg != NULL) {
 493         char *s = crm_strdup_printf("bans:%s", optarg);
 494         gboolean rc = include_exclude_cb("--include", s, data, err);
 495         free(s);
 496         return rc;
 497     } else {
 498         return include_exclude_cb("--include", "bans", data, err);
 499     }
 500 }
 501 
 502 static gboolean
 503 show_failcounts_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 504     return include_exclude_cb("--include", "failcounts", data, err);
 505 }
 506 
 507 static gboolean
 508 show_operations_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 509     return include_exclude_cb("--include", "failcounts,operations", data, err);
 510 }
 511 
 512 static gboolean
 513 show_tickets_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 514     return include_exclude_cb("--include", "tickets", data, err);
 515 }
 516 
 517 static gboolean
 518 use_cib_file_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 519     setenv("CIB_file", optarg, 1);
 520     options.mon_ops |= mon_op_one_shot;
 521     return TRUE;
 522 }
 523 
 524 static gboolean
 525 watch_fencing_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /*  */
 526     options.mon_ops |= mon_op_watch_fencing;
 527     return TRUE;
 528 }
 529 
 530 #define INDENT "                                    "
 531 
 532 /* *INDENT-OFF* */
 533 static GOptionEntry addl_entries[] = {
 534     { "interval", 'i', 0, G_OPTION_ARG_CALLBACK, reconnect_cb,
 535       "Update frequency (default is 5 seconds)",
 536       "TIMESPEC" },
 537 
 538     { "one-shot", '1', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, one_shot_cb,
 539       "Display the cluster status once on the console and exit",
 540       NULL },
 541 
 542     { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &options.daemonize,
 543       "Run in the background as a daemon.\n"
 544       INDENT "Requires at least one of --output-to and --external-agent.",
 545       NULL },
 546 
 547     { "pid-file", 'p', 0, G_OPTION_ARG_FILENAME, &options.pid_file,
 548       "(Advanced) Daemon pid file location",
 549       "FILE" },
 550 
 551     { "external-agent", 'E', 0, G_OPTION_ARG_FILENAME, &options.external_agent,
 552       "A program to run when resource operations take place",
 553       "FILE" },
 554 
 555     { "external-recipient", 'e', 0, G_OPTION_ARG_STRING, &options.external_recipient,
 556       "A recipient for your program (assuming you want the program to send something to someone).",
 557       "RCPT" },
 558 
 559     { "watch-fencing", 'W', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, watch_fencing_cb,
 560       "Listen for fencing events. For use with --external-agent.",
 561       NULL },
 562 
 563     { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, use_cib_file_cb,
 564       NULL,
 565       NULL },
 566 
 567     { NULL }
 568 };
 569 
 570 static GOptionEntry display_entries[] = {
 571     { "include", 'I', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 572       "A list of sections to include in the output.\n"
 573       INDENT "See `Output Control` help for more information.",
 574       "SECTION(s)" },
 575 
 576     { "exclude", 'U', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 577       "A list of sections to exclude from the output.\n"
 578       INDENT "See `Output Control` help for more information.",
 579       "SECTION(s)" },
 580 
 581     { "node", 0, 0, G_OPTION_ARG_STRING, &options.only_node,
 582       "When displaying information about nodes, show only what's related to the given\n"
 583       INDENT "node, or to all nodes tagged with the given tag",
 584       "NODE" },
 585 
 586     { "resource", 0, 0, G_OPTION_ARG_STRING, &options.only_rsc,
 587       "When displaying information about resources, show only what's related to the given\n"
 588       INDENT "resource, or to all resources tagged with the given tag",
 589       "RSC" },
 590 
 591     { "group-by-node", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, group_by_node_cb,
 592       "Group resources by node",
 593       NULL },
 594 
 595     { "inactive", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, inactive_resources_cb,
 596       "Display inactive resources",
 597       NULL },
 598 
 599     { "failcounts", 'f', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_failcounts_cb,
 600       "Display resource fail counts",
 601       NULL },
 602 
 603     { "operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_operations_cb,
 604       "Display resource operation history",
 605       NULL },
 606 
 607     { "timing-details", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_timing_cb,
 608       "Display resource operation history with timing details",
 609       NULL },
 610 
 611     { "tickets", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_tickets_cb,
 612       "Display cluster tickets",
 613       NULL },
 614 
 615     { "fence-history", 'm', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, fence_history_cb,
 616       "Show fence history:\n"
 617       INDENT "0=off, 1=failures and pending (default without option),\n"
 618       INDENT "2=add successes (default without value for option),\n"
 619       INDENT "3=show full history without reduction to most recent of each flavor",
 620       "LEVEL" },
 621 
 622     { "neg-locations", 'L', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, show_bans_cb,
 623       "Display negative location constraints [optionally filtered by id prefix]",
 624       NULL },
 625 
 626     { "show-node-attributes", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_attributes_cb,
 627       "Display node attributes",
 628       NULL },
 629 
 630     { "hide-headers", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, hide_headers_cb,
 631       "Hide all headers",
 632       NULL },
 633 
 634     { "show-detail", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_clone_detail_cb,
 635       "Show more details (node IDs, individual clone instances)",
 636       NULL },
 637 
 638     { "brief", 'b', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_brief_cb,
 639       "Brief output",
 640       NULL },
 641 
 642     { "pending", 'j', G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_pending_cb,
 643       "Display pending state if 'record-pending' is enabled",
 644       NULL },
 645 
 646     { "simple-status", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_simple_cb,
 647       "Display the cluster status once as a simple one line output (suitable for nagios)",
 648       NULL },
 649 
 650     { NULL }
 651 };
 652 
 653 static GOptionEntry deprecated_entries[] = {
 654     { "as-html", 'h', G_OPTION_FLAG_FILENAME, G_OPTION_ARG_CALLBACK, as_html_cb,
 655       "Write cluster status to the named HTML file.\n"
 656       INDENT "Use --output-as=html --output-to=FILE instead.",
 657       "FILE" },
 658 
 659     { "as-xml", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_xml_cb,
 660       "Write cluster status as XML to stdout. This will enable one-shot mode.\n"
 661       INDENT "Use --output-as=xml instead.",
 662       NULL },
 663 
 664     { "disable-ncurses", 'N', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, no_curses_cb,
 665       "Disable the use of ncurses.\n"
 666       INDENT "Use --output-as=text instead.",
 667       NULL },
 668 
 669     { "web-cgi", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_cgi_cb,
 670       "Web mode with output suitable for CGI (preselected when run as *.cgi).\n"
 671       INDENT "Use --output-as=html --html-cgi instead.",
 672       NULL },
 673 
 674     { NULL }
 675 };
 676 /* *INDENT-ON* */
 677 
 678 /*!
 679  * \internal
 680  * \brief Return resource display options corresponding to command-line choices
 681  *
 682  * \return Bitmask of pe_print_options suitable for resource print functions
 683  */
 684 static unsigned int
 685 get_resource_display_options(unsigned int mon_ops)
     /*  */
 686 {
 687     int print_opts = 0;
 688 
 689     if (pcmk_is_set(mon_ops, mon_op_print_pending)) {
 690         print_opts |= pe_print_pending;
 691     }
 692     if (pcmk_is_set(mon_ops, mon_op_print_clone_detail)) {
 693         print_opts |= pe_print_clone_details|pe_print_implicit;
 694     }
 695     if (!pcmk_is_set(mon_ops, mon_op_inactive_resources)) {
 696         print_opts |= pe_print_clone_active;
 697     }
 698     if (pcmk_is_set(mon_ops, mon_op_print_brief)) {
 699         print_opts |= pe_print_brief;
 700     }
 701     return print_opts;
 702 }
 703 
 704 static void
 705 blank_screen(void)
     /*  */
 706 {
 707 #if CURSES_ENABLED
 708     int lpc = 0;
 709 
 710     for (lpc = 0; lpc < LINES; lpc++) {
 711         move(lpc, 0);
 712         clrtoeol();
 713     }
 714     move(0, 0);
 715     refresh();
 716 #endif
 717 }
 718 
 719 /* Reconnect to the CIB and fencing agent after reconnect_ms has passed.  This sounds
 720  * like it would be more broadly useful, but only ever happens after a disconnect via
 721  * mon_cib_connection_destroy.
 722  */
 723 static gboolean
 724 reconnect_after_timeout(gpointer data)
     /*  */
 725 {
 726 #if CURSES_ENABLED
 727     if (output_format == mon_output_console) {
 728         clear();
 729         refresh();
 730     }
 731 #endif
 732 
 733     out->info(out, "Reconnecting...");
 734     if (pacemakerd_status() == pcmk_rc_ok) {
 735         fencing_connect();
 736         if (cib_connect(TRUE) == pcmk_rc_ok) {
 737             /* trigger redrawing the screen (needs reconnect_timer == 0) */
 738             reconnect_timer = 0;
 739             refresh_after_event(FALSE, TRUE);
 740             return G_SOURCE_REMOVE;
 741         }
 742     }
 743 
 744     reconnect_timer = g_timeout_add(options.reconnect_ms,
 745                                     reconnect_after_timeout, NULL);
 746     return G_SOURCE_REMOVE;
 747 }
 748 
 749 /* Called from various places when we are disconnected from the CIB or from the
 750  * fencing agent.  If the CIB connection is still valid, this function will also
 751  * attempt to sign off and reconnect.
 752  */
 753 static void
 754 mon_cib_connection_destroy(gpointer user_data)
     /*  */
 755 {
 756     out->info(out, "Connection to the cluster-daemons terminated");
 757 
 758     if (refresh_timer != NULL) {
 759         /* we'll trigger a refresh after reconnect */
 760         mainloop_timer_stop(refresh_timer);
 761     }
 762     if (reconnect_timer) {
 763         /* we'll trigger a new reconnect-timeout at the end */
 764         g_source_remove(reconnect_timer);
 765         reconnect_timer = 0;
 766     }
 767     if (st) {
 768         /* the client API won't properly reconnect notifications
 769          * if they are still in the table - so remove them
 770          */
 771         clean_up_fencing_connection();
 772     }
 773     if (cib) {
 774         cib->cmds->signoff(cib);
 775         reconnect_timer = g_timeout_add(options.reconnect_ms,
 776                                         reconnect_after_timeout, NULL);
 777     }
 778     return;
 779 }
 780 
 781 /* Signal handler installed into the mainloop for normal program shutdown */
 782 static void
 783 mon_shutdown(int nsig)
     /*  */
 784 {
 785     clean_up(CRM_EX_OK);
 786 }
 787 
 788 #if CURSES_ENABLED
 789 static volatile sighandler_t ncurses_winch_handler;
 790 
 791 /* Signal handler installed the regular way (not into the main loop) for when
 792  * the screen is resized.  Commonly, this happens when running in an xterm and
 793  * the user changes its size.
 794  */
 795 static void
 796 mon_winresize(int nsig)
     /*  */
 797 {
 798     static int not_done;
 799     int lines = 0, cols = 0;
 800 
 801     if (!not_done++) {
 802         if (ncurses_winch_handler)
 803             /* the original ncurses WINCH signal handler does the
 804              * magic of retrieving the new window size;
 805              * otherwise, we'd have to use ioctl or tgetent */
 806             (*ncurses_winch_handler) (SIGWINCH);
 807         getmaxyx(stdscr, lines, cols);
 808         resizeterm(lines, cols);
 809         /* Alert the mainloop code we'd like the refresh_trigger to run next
 810          * time the mainloop gets around to checking.
 811          */
 812         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
 813     }
 814     not_done--;
 815 }
 816 #endif
 817 
 818 static int
 819 fencing_connect(void)
     /*  */
 820 {
 821     int rc = pcmk_ok;
 822 
 823     if (pcmk_is_set(options.mon_ops, mon_op_fence_connect) && (st == NULL)) {
 824         st = stonith_api_new();
 825     }
 826 
 827     if (!pcmk_is_set(options.mon_ops, mon_op_fence_connect) ||
 828         st == NULL || st->state != stonith_disconnected) {
 829         return rc;
 830     }
 831 
 832     rc = st->cmds->connect(st, crm_system_name, NULL);
 833     if (rc == pcmk_ok) {
 834         crm_trace("Setting up stonith callbacks");
 835         if (pcmk_is_set(options.mon_ops, mon_op_watch_fencing)) {
 836             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 837                                             mon_st_callback_event);
 838             st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback_event);
 839         } else {
 840             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 841                                             mon_st_callback_display);
 842             st->cmds->register_notification(st, T_STONITH_NOTIFY_HISTORY, mon_st_callback_display);
 843         }
 844     } else {
 845         clean_up_fencing_connection();
 846     }
 847 
 848     return rc;
 849 }
 850 
 851 static int
 852 cib_connect(gboolean full)
     /*  */
 853 {
 854     int rc = pcmk_rc_ok;
 855 
 856     CRM_CHECK(cib != NULL, return EINVAL);
 857 
 858     if (cib->state == cib_connected_query ||
 859         cib->state == cib_connected_command) {
 860         return rc;
 861     }
 862 
 863     crm_trace("Connecting to the CIB");
 864 
 865     rc = pcmk_legacy2rc(cib->cmds->signon(cib, crm_system_name, cib_query));
 866     if (rc != pcmk_rc_ok) {
 867         out->err(out, "Could not connect to the CIB: %s",
 868                  pcmk_rc_str(rc));
 869         return rc;
 870     }
 871 
 872 #if CURSES_ENABLED
 873     /* just show this if refresh is gonna remove all traces */
 874     if (output_format == mon_output_console) {
 875         out->info(out,"Waiting for CIB ...");
 876     }
 877 #endif
 878 
 879     rc = pcmk_legacy2rc(cib->cmds->query(cib, NULL, &current_cib,
 880                                          cib_scope_local | cib_sync_call));
 881 
 882     if (rc == pcmk_rc_ok && full) {
 883         rc = pcmk_legacy2rc(cib->cmds->set_connection_dnotify(cib,
 884             mon_cib_connection_destroy));
 885         if (rc == EPROTONOSUPPORT) {
 886             out->err(out,
 887                      "Notification setup not supported, won't be "
 888                      "able to reconnect after failure");
 889             if (output_format == mon_output_console) {
 890                 sleep(2);
 891             }
 892             rc = pcmk_rc_ok;
 893         }
 894 
 895         if (rc == pcmk_rc_ok) {
 896             cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY,
 897                                            crm_diff_update);
 898             rc = pcmk_legacy2rc(cib->cmds->add_notify_callback(cib,
 899                                     T_CIB_DIFF_NOTIFY, crm_diff_update));
 900         }
 901 
 902         if (rc != pcmk_rc_ok) {
 903             out->err(out, "Notification setup failed, could not monitor CIB actions");
 904             clean_up_cib_connection();
 905             clean_up_fencing_connection();
 906         }
 907     }
 908     return rc;
 909 }
 910 
 911 /* This is used to set up the fencing options after the interactive UI has been stared.
 912  * fence_history_cb can't be used because it builds up a list of includes/excludes that
 913  * then have to be processed with apply_include_exclude and that could affect other
 914  * things.
 915  */
 916 static void
 917 set_fencing_options(int level)
     /*  */
 918 {
 919     switch (level) {
 920         case 3:
 921             options.mon_ops |= mon_op_fence_full_history | mon_op_fence_history | mon_op_fence_connect;
 922             show |= mon_show_fencing_all;
 923             break;
 924 
 925         case 2:
 926             options.mon_ops |= mon_op_fence_history | mon_op_fence_connect;
 927             show |= mon_show_fencing_all;
 928             break;
 929 
 930         case 1:
 931             options.mon_ops |= mon_op_fence_history | mon_op_fence_connect;
 932             show |= mon_show_fence_failed | mon_show_fence_pending;
 933             break;
 934 
 935         default:
 936             interactive_fence_level = 0;
 937             options.mon_ops &= ~(mon_op_fence_history | mon_op_fence_connect);
 938             show &= ~mon_show_fencing_all;
 939             break;
 940     }
 941 }
 942 
 943 /* Before trying to connect to fencer or cib check for state of
 944    pacemakerd - just no sense in trying till pacemakerd has
 945    taken care of starting all the sub-processes
 946 
 947    Only noteworthy thing to show here is when pacemakerd is
 948    waiting for startup-trigger from SBD.
 949  */
 950 static void
 951 pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
     /*  */
 952                     enum pcmk_ipc_event event_type, crm_exit_t status,
 953                     void *event_data, void *user_data)
 954 {
 955     pcmk_pacemakerd_api_reply_t *reply = event_data;
 956     enum pcmk_pacemakerd_state *state =
 957         (enum pcmk_pacemakerd_state *) user_data;
 958 
 959     /* we are just interested in the latest reply */
 960     *state = pcmk_pacemakerd_state_invalid;
 961 
 962     switch (event_type) {
 963         case pcmk_ipc_event_reply:
 964             break;
 965 
 966         default:
 967             return;
 968     }
 969 
 970     if (status != CRM_EX_OK) {
 971         out->err(out, "Bad reply from pacemakerd: %s",
 972                  crm_exit_str(status));
 973         return;
 974     }
 975 
 976     if (reply->reply_type != pcmk_pacemakerd_reply_ping) {
 977         out->err(out, "Unknown reply type %d from pacemakerd",
 978                  reply->reply_type);
 979     } else {
 980         if ((reply->data.ping.last_good != (time_t) 0) &&
 981             (reply->data.ping.status == pcmk_rc_ok)) {
 982             *state = reply->data.ping.state;
 983         }
 984     }
 985 }
 986 
 987 static int
 988 pacemakerd_status(void)
     /*  */
 989 {
 990     int rc = pcmk_rc_ok;
 991     pcmk_ipc_api_t *pacemakerd_api = NULL;
 992     enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;
 993 
 994     if (!pcmk_is_set(options.mon_ops, mon_op_cib_native)) {
 995         /* we don't need fully functional pacemakerd otherwise */
 996         return rc;
 997     }
 998     if (cib != NULL &&
 999         (cib->state == cib_connected_query ||
1000          cib->state == cib_connected_command)) {
1001         /* As long as we have a cib-connection let's go with
1002          * that to fetch further cluster-status and avoid
1003          * unnecessary pings to pacemakerd.
1004          * If cluster is going down and fencer is down already
1005          * this will lead to a silently failing fencer reconnect.
1006          * On cluster startup we shouldn't see this situation
1007          * as first we do is wait for pacemakerd to report all
1008          * daemons running.
1009          */
1010         return rc;
1011     }
1012     rc = pcmk_new_ipc_api(&pacemakerd_api, pcmk_ipc_pacemakerd);
1013     if (pacemakerd_api == NULL) {
1014         out->err(out, "Could not connect to pacemakerd: %s",
1015                  pcmk_rc_str(rc));
1016         /* this is unrecoverable so return with rc we have */
1017         return rc;
1018     }
1019     pcmk_register_ipc_callback(pacemakerd_api, pacemakerd_event_cb, (void *) &state);
1020     rc = pcmk_connect_ipc(pacemakerd_api, pcmk_ipc_dispatch_poll);
1021     switch (rc) {
1022         case pcmk_rc_ok:
1023             rc = pcmk_pacemakerd_api_ping(pacemakerd_api, crm_system_name);
1024             if (rc == pcmk_rc_ok) {
1025                 rc = pcmk_poll_ipc(pacemakerd_api, options.reconnect_ms/2);
1026                 if (rc == pcmk_rc_ok) {
1027                     pcmk_dispatch_ipc(pacemakerd_api);
1028                     rc = ENOTCONN;
1029                     if ((output_format == mon_output_console) ||
1030                         (output_format == mon_output_plain)) {
1031                         switch (state) {
1032                             case pcmk_pacemakerd_state_running:
1033                                 rc = pcmk_rc_ok;
1034                                 break;
1035                             case pcmk_pacemakerd_state_starting_daemons:
1036                                 out->info(out,"Pacemaker daemons starting ...");
1037                                 break;
1038                             case pcmk_pacemakerd_state_wait_for_ping:
1039                                 out->info(out,"Waiting for startup-trigger from SBD ...");
1040                                 break;
1041                             case pcmk_pacemakerd_state_shutting_down:
1042                                 out->info(out,"Pacemaker daemons shutting down ...");
1043                                 /* try our luck maybe CIB is still accessible */
1044                                 rc = pcmk_rc_ok;
1045                                 break;
1046                             case pcmk_pacemakerd_state_shutdown_complete:
1047                                 /* assuming pacemakerd doesn't dispatch any pings after entering
1048                                 * that state unless it is waiting for SBD
1049                                 */
1050                                 out->info(out,"Pacemaker daemons shut down - reporting to SBD ...");
1051                                 break;
1052                             default:
1053                                 break;
1054                         }
1055                     } else {
1056                         switch (state) {
1057                             case pcmk_pacemakerd_state_running:
1058                                 rc = pcmk_rc_ok;
1059                                 break;
1060                             case pcmk_pacemakerd_state_shutting_down:
1061                                 /* try our luck maybe CIB is still accessible */
1062                                 rc = pcmk_rc_ok;
1063                                 break;
1064                             default:
1065                                 break;
1066                         }
1067                     }
1068                 }
1069             }
1070             break;
1071         case EREMOTEIO:
1072             rc = pcmk_rc_ok;
1073             on_remote_node = TRUE;
1074 #if CURSES_ENABLED
1075             /* just show this if refresh is gonna remove all traces */
1076             if (output_format == mon_output_console) {
1077                 out->info(out,
1078                     "Running on remote-node waiting to be connected by cluster ...");
1079             }
1080 #endif
1081             break;
1082         default:
1083             break;
1084     }
1085     pcmk_free_ipc_api(pacemakerd_api);
1086     /* returning with ENOTCONN triggers a retry */
1087     return (rc == pcmk_rc_ok)?rc:ENOTCONN;
1088 }
1089 
1090 #if CURSES_ENABLED
1091 static const char *
1092 get_option_desc(char c)
     /*  */
1093 {
1094     const char *desc = "No help available";
1095 
1096     for (GOptionEntry *entry = display_entries; entry != NULL; entry++) {
1097         if (entry->short_name == c) {
1098             desc = entry->description;
1099             break;
1100         }
1101     }
1102     return desc;
1103 }
1104 
1105 #define print_option_help(out, option, condition) \
1106     curses_formatted_printf(out, "%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
1107 
1108 /* This function is called from the main loop when there is something to be read
1109  * on stdin, like an interactive user's keystroke.  All it does is read the keystroke,
1110  * set flags (or show the page showing which keystrokes are valid), and redraw the
1111  * screen.  It does not do anything with connections to the CIB or fencing agent
1112  * agent what would happen in mon_refresh_display.
1113  */
1114 static gboolean
1115 detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer user_data)
     /*  */
1116 {
1117     int c;
1118     gboolean config_mode = FALSE;
1119 
1120     while (1) {
1121 
1122         /* Get user input */
1123         c = getchar();
1124 
1125         switch (c) {
1126             case 'm':
1127                 interactive_fence_level++;
1128                 if (interactive_fence_level > 3) {
1129                     interactive_fence_level = 0;
1130                 }
1131 
1132                 set_fencing_options(interactive_fence_level);
1133                 break;
1134             case 'c':
1135                 show ^= mon_show_tickets;
1136                 break;
1137             case 'f':
1138                 show ^= mon_show_failcounts;
1139                 break;
1140             case 'n':
1141                 options.mon_ops ^= mon_op_group_by_node;
1142                 break;
1143             case 'o':
1144                 show ^= mon_show_operations;
1145                 if (!pcmk_is_set(show, mon_show_operations)) {
1146                     options.mon_ops &= ~mon_op_print_timing;
1147                 }
1148                 break;
1149             case 'r':
1150                 options.mon_ops ^= mon_op_inactive_resources;
1151                 break;
1152             case 'R':
1153                 options.mon_ops ^= mon_op_print_clone_detail;
1154                 break;
1155             case 't':
1156                 options.mon_ops ^= mon_op_print_timing;
1157                 if (pcmk_is_set(options.mon_ops, mon_op_print_timing)) {
1158                     show |= mon_show_operations;
1159                 }
1160                 break;
1161             case 'A':
1162                 show ^= mon_show_attributes;
1163                 break;
1164             case 'L':
1165                 show ^= mon_show_bans;
1166                 break;
1167             case 'D':
1168                 /* If any header is shown, clear them all, otherwise set them all */
1169                 if (pcmk_any_flags_set(show,
1170                                        mon_show_stack
1171                                        |mon_show_dc
1172                                        |mon_show_times
1173                                        |mon_show_counts)) {
1174                     show &= ~mon_show_summary;
1175                 } else {
1176                     show |= mon_show_summary;
1177                 }
1178                 /* Regardless, we don't show options in console mode. */
1179                 show &= ~mon_show_options;
1180                 break;
1181             case 'b':
1182                 options.mon_ops ^= mon_op_print_brief;
1183                 break;
1184             case 'j':
1185                 options.mon_ops ^= mon_op_print_pending;
1186                 break;
1187             case '?':
1188                 config_mode = TRUE;
1189                 break;
1190             default:
1191                 /* All other keys just redraw the screen. */
1192                 goto refresh;
1193         }
1194 
1195         if (!config_mode)
1196             goto refresh;
1197 
1198         blank_screen();
1199 
1200         curses_formatted_printf(out, "%s", "Display option change mode\n");
1201         print_option_help(out, 'c', pcmk_is_set(show, mon_show_tickets));
1202         print_option_help(out, 'f', pcmk_is_set(show, mon_show_failcounts));
1203         print_option_help(out, 'n', pcmk_is_set(options.mon_ops, mon_op_group_by_node));
1204         print_option_help(out, 'o', pcmk_is_set(show, mon_show_operations));
1205         print_option_help(out, 'r', pcmk_is_set(options.mon_ops, mon_op_inactive_resources));
1206         print_option_help(out, 't', pcmk_is_set(options.mon_ops, mon_op_print_timing));
1207         print_option_help(out, 'A', pcmk_is_set(show, mon_show_attributes));
1208         print_option_help(out, 'L', pcmk_is_set(show,mon_show_bans));
1209         print_option_help(out, 'D', !pcmk_is_set(show, mon_show_summary));
1210         print_option_help(out, 'R', pcmk_is_set(options.mon_ops, mon_op_print_clone_detail));
1211         print_option_help(out, 'b', pcmk_is_set(options.mon_ops, mon_op_print_brief));
1212         print_option_help(out, 'j', pcmk_is_set(options.mon_ops, mon_op_print_pending));
1213         curses_formatted_printf(out, "%d m: \t%s\n", interactive_fence_level, get_option_desc('m'));
1214         curses_formatted_printf(out, "%s", "\nToggle fields via field letter, type any other key to return\n");
1215     }
1216 
1217 refresh:
1218     refresh_after_event(FALSE, TRUE);
1219 
1220     return TRUE;
1221 }
1222 #endif
1223 
1224 // Basically crm_signal_handler(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
1225 static void
1226 avoid_zombies(void)
     /*  */
1227 {
1228     struct sigaction sa;
1229 
1230     memset(&sa, 0, sizeof(struct sigaction));
1231     if (sigemptyset(&sa.sa_mask) < 0) {
1232         crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
1233         return;
1234     }
1235     sa.sa_handler = SIG_IGN;
1236     sa.sa_flags = SA_RESTART|SA_NOCLDWAIT;
1237     if (sigaction(SIGCHLD, &sa, NULL) < 0) {
1238         crm_warn("Cannot avoid zombies: %s", pcmk_strerror(errno));
1239     }
1240 }
1241 
1242 static GOptionContext *
1243 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     /*  */
1244     GOptionContext *context = NULL;
1245 
1246     GOptionEntry extra_prog_entries[] = {
1247         { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
1248           "Be less descriptive in output.",
1249           NULL },
1250 
1251         { NULL }
1252     };
1253 
1254     const char *description = "Notes:\n\n"
1255                               "If this program is called as crm_mon.cgi, --output-as=html --html-cgi will\n"
1256                               "automatically be added to the command line arguments.\n\n"
1257                               "Time Specification:\n\n"
1258                               "The TIMESPEC in any command line option can be specified in many different\n"
1259                               "formats.  It can be just an integer number of seconds, a number plus units\n"
1260                               "(ms/msec/us/usec/s/sec/m/min/h/hr), or an ISO 8601 period specification.\n\n"
1261                               "Output Control:\n\n"
1262                               "By default, a certain list of sections are written to the output destination.\n"
1263                               "The default varies based on the output format - XML includes everything, while\n"
1264                               "other output formats will display less.  This list can be modified with the\n"
1265                               "--include and --exclude command line options.  Each option may be given multiple\n"
1266                               "times on the command line, and each can give a comma-separated list of sections.\n"
1267                               "The options are applied to the default set, from left to right as seen on the\n"
1268                               "command line.  For a list of valid sections, pass --include=list or --exclude=list.\n\n"
1269                               "Interactive Use:\n\n"
1270                               "When run interactively, crm_mon can be told to hide and display various sections\n"
1271                               "of output.  To see a help screen explaining the options, hit '?'.  Any key stroke\n"
1272                               "aside from those listed will cause the screen to refresh.\n\n"
1273                               "Examples:\n\n"
1274                               "Display the cluster status on the console with updates as they occur:\n\n"
1275                               "\tcrm_mon\n\n"
1276                               "Display the cluster status on the console just once then exit:\n\n"
1277                               "\tcrm_mon -1\n\n"
1278                               "Display your cluster status, group resources by node, and include inactive resources in the list:\n\n"
1279                               "\tcrm_mon --group-by-node --inactive\n\n"
1280                               "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:\n\n"
1281                               "\tcrm_mon --daemonize --output-as html --output-to /path/to/docroot/filename.html\n\n"
1282                               "Start crm_mon and export the current cluster status as XML to stdout, then exit:\n\n"
1283                               "\tcrm_mon --output-as xml\n\n";
1284 
1285     context = pcmk__build_arg_context(args, "console (default), html, text, xml", group, NULL);
1286     pcmk__add_main_args(context, extra_prog_entries);
1287     g_option_context_set_description(context, description);
1288 
1289     pcmk__add_arg_group(context, "display", "Display Options:",
1290                         "Show display options", display_entries);
1291     pcmk__add_arg_group(context, "additional", "Additional Options:",
1292                         "Show additional options", addl_entries);
1293     pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
1294                         "Show deprecated options", deprecated_entries);
1295 
1296     return context;
1297 }
1298 
1299 /* If certain format options were specified, we want to set some extra
1300  * options.  We can just process these like they were given on the
1301  * command line.
1302  */
1303 static void
1304 add_output_args(void) {
     /*  */
1305     GError *err = NULL;
1306 
1307     if (output_format == mon_output_plain) {
1308         if (!pcmk__force_args(context, &err, "%s --text-fancy", g_get_prgname())) {
1309             g_propagate_error(&error, err);
1310             clean_up(CRM_EX_USAGE);
1311         }
1312     } else if (output_format == mon_output_cgi) {
1313         if (!pcmk__force_args(context, &err, "%s --html-cgi", g_get_prgname())) {
1314             g_propagate_error(&error, err);
1315             clean_up(CRM_EX_USAGE);
1316         }
1317     } else if (output_format == mon_output_xml) {
1318         if (!pcmk__force_args(context, &err, "%s --xml-simple-list --xml-substitute", g_get_prgname())) {
1319             g_propagate_error(&error, err);
1320             clean_up(CRM_EX_USAGE);
1321         }
1322     } else if (output_format == mon_output_legacy_xml) {
1323         output_format = mon_output_xml;
1324         if (!pcmk__force_args(context, &err, "%s --xml-legacy --xml-substitute", g_get_prgname())) {
1325             g_propagate_error(&error, err);
1326             clean_up(CRM_EX_USAGE);
1327         }
1328     }
1329 }
1330 
1331 /* Which output format to use could come from two places:  The --as-xml
1332  * style arguments we gave in deprecated_entries above, or the formatted output
1333  * arguments added by pcmk__register_formats.  If the latter were used,
1334  * output_format will be mon_output_unset.
1335  *
1336  * Call the callbacks as if those older style arguments were provided so
1337  * the various things they do get done.
1338  */
1339 static void
1340 reconcile_output_format(pcmk__common_args_t *args) {
     /*  */
1341     gboolean retval = TRUE;
1342     GError *err = NULL;
1343 
1344     if (output_format != mon_output_unset) {
1345         return;
1346     }
1347 
1348     if (pcmk__str_eq(args->output_ty, "html", pcmk__str_casei)) {
1349         char *dest = NULL;
1350 
1351         if (args->output_dest != NULL) {
1352             dest = strdup(args->output_dest);
1353         }
1354 
1355         retval = as_html_cb("h", dest, NULL, &err);
1356         free(dest);
1357     } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_casei)) {
1358         retval = no_curses_cb("N", NULL, NULL, &err);
1359     } else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_casei)) {
1360         if (args->output_ty != NULL) {
1361             free(args->output_ty);
1362         }
1363 
1364         args->output_ty = strdup("xml");
1365         output_format = mon_output_xml;
1366     } else if (pcmk_is_set(options.mon_ops, mon_op_one_shot)) {
1367         if (args->output_ty != NULL) {
1368             free(args->output_ty);
1369         }
1370 
1371         args->output_ty = strdup("text");
1372         output_format = mon_output_plain;
1373     } else {
1374         /* Neither old nor new arguments were given, so set the default. */
1375         if (args->output_ty != NULL) {
1376             free(args->output_ty);
1377         }
1378 
1379         args->output_ty = strdup("console");
1380         output_format = mon_output_console;
1381     }
1382 
1383     if (!retval) {
1384         g_propagate_error(&error, err);
1385         clean_up(CRM_EX_USAGE);
1386     }
1387 }
1388 
1389 static void
1390 handle_connection_failures(int rc)
     /*  */
1391 {
1392     if (rc == pcmk_rc_ok) {
1393         return;
1394     }
1395 
1396     if (output_format == mon_output_monitor) {
1397         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "CLUSTER CRIT: Connection to cluster failed: %s",
1398                     pcmk_rc_str(rc));
1399         rc = MON_STATUS_CRIT;
1400     } else if (rc == ENOTCONN) {
1401         if (on_remote_node) {
1402             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: remote-node not connected to cluster");
1403         } else {
1404             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");
1405         }
1406         rc = pcmk_rc2exitc(rc);
1407     } else {
1408         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Connection to cluster failed: %s", pcmk_rc_str(rc));
1409         rc = pcmk_rc2exitc(rc);
1410     }
1411 
1412     clean_up(rc);
1413 }
1414 
1415 static void
1416 one_shot(void)
     /*  */
1417 {
1418     int rc;
1419 
1420     rc = pacemakerd_status();
1421 
1422     if (rc == pcmk_rc_ok) {
1423         fencing_connect();
1424         rc = cib_connect(FALSE);
1425     }
1426 
1427     if (rc == pcmk_rc_ok) {
1428         mon_refresh_display(NULL);
1429     } else {
1430         handle_connection_failures(rc);
1431     }
1432 
1433     clean_up(CRM_EX_OK);
1434 }
1435 
1436 int
1437 main(int argc, char **argv)
     /*  */
1438 {
1439     int rc = pcmk_ok;
1440     GOptionGroup *output_group = NULL;
1441 
1442     args = pcmk__new_common_args(SUMMARY);
1443     context = build_arg_context(args, &output_group);
1444     pcmk__register_formats(output_group, formats);
1445 
1446     options.pid_file = strdup("/tmp/ClusterMon.pid");
1447     pcmk__cli_init_logging("crm_mon", 0);
1448 
1449     // Avoid needing to wait for subprocesses forked for -E/--external-agent
1450     avoid_zombies();
1451 
1452     if (pcmk__ends_with_ext(argv[0], ".cgi")) {
1453         output_format = mon_output_cgi;
1454         options.mon_ops |= mon_op_one_shot;
1455     }
1456 
1457     processed_args = pcmk__cmdline_preproc(argv, "ehimpxEILU");
1458 
1459     fence_history_cb("--fence-history", "1", NULL, NULL);
1460 
1461     /* Set an HTML title regardless of what format we will eventually use.  This can't
1462      * be done in add_output_args.  That function is called after command line
1463      * arguments are processed in the next block, which means it'll override whatever
1464      * title the user provides.  Doing this here means the user can give their own
1465      * title on the command line.
1466      */
1467     if (!pcmk__force_args(context, &error, "%s --html-title \"Cluster Status\"",
1468                           g_get_prgname())) {
1469         return clean_up(CRM_EX_USAGE);
1470     }
1471 
1472     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
1473         return clean_up(CRM_EX_USAGE);
1474     }
1475 
1476     for (int i = 0; i < args->verbosity; i++) {
1477         crm_bump_log_level(argc, argv);
1478     }
1479 
1480     if (!args->version) {
1481         if (args->quiet) {
1482             include_exclude_cb("--exclude", "times", NULL, NULL);
1483         }
1484 
1485         if (pcmk_is_set(options.mon_ops, mon_op_watch_fencing)) {
1486             fence_history_cb("--fence-history", "0", NULL, NULL);
1487             options.mon_ops |= mon_op_fence_connect;
1488         }
1489 
1490         /* create the cib-object early to be able to do further
1491          * decisions based on the cib-source
1492          */
1493         cib = cib_new();
1494 
1495         if (cib == NULL) {
1496             rc = -EINVAL;
1497         } else {
1498             switch (cib->variant) {
1499 
1500                 case cib_native:
1501                     /* cib & fencing - everything available */
1502                     options.mon_ops |= mon_op_cib_native;
1503                     break;
1504 
1505                 case cib_file:
1506                     /* Don't try to connect to fencing as we
1507                      * either don't have a running cluster or
1508                      * the fencing-information would possibly
1509                      * not match the cib data from a file.
1510                      * As we don't expect cib-updates coming
1511                      * in enforce one-shot. */
1512                     fence_history_cb("--fence-history", "0", NULL, NULL);
1513                     options.mon_ops |= mon_op_one_shot;
1514                     break;
1515 
1516                 case cib_remote:
1517                     /* updates coming in but no fencing */
1518                     fence_history_cb("--fence-history", "0", NULL, NULL);
1519                     break;
1520 
1521                 case cib_undefined:
1522                 case cib_database:
1523                 default:
1524                     /* something is odd */
1525                     rc = -EINVAL;
1526                     break;
1527             }
1528         }
1529 
1530         if (pcmk_is_set(options.mon_ops, mon_op_one_shot)) {
1531             if (output_format == mon_output_console) {
1532                 output_format = mon_output_plain;
1533             }
1534 
1535         } else if (options.daemonize) {
1536             if ((output_format == mon_output_console) || (args->output_dest == NULL)) {
1537                 output_format = mon_output_none;
1538             }
1539             crm_enable_stderr(FALSE);
1540 
1541             if (pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches | pcmk__str_casei) && !options.external_agent) {
1542                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "--daemonize requires at least one of --output-to and --external-agent");
1543                 return clean_up(CRM_EX_USAGE);
1544             }
1545 
1546             if (cib) {
1547                 /* to be on the safe side don't have cib-object around
1548                  * when we are forking
1549                  */
1550                 cib_delete(cib);
1551                 cib = NULL;
1552                 pcmk__daemonize(crm_system_name, options.pid_file);
1553                 cib = cib_new();
1554                 if (cib == NULL) {
1555                     rc = -EINVAL;
1556                 }
1557                 /* otherwise assume we've got the same cib-object we've just destroyed
1558                  * in our parent
1559                  */
1560             }
1561 
1562 
1563         } else if (output_format == mon_output_console) {
1564 #if CURSES_ENABLED
1565             crm_enable_stderr(FALSE);
1566 #else
1567             options.mon_ops |= mon_op_one_shot;
1568             output_format = mon_output_plain;
1569             printf("Defaulting to one-shot mode\n");
1570             printf("You need to have curses available at compile time to enable console mode\n");
1571 #endif
1572         }
1573     }
1574 
1575     if (rc != pcmk_ok) {
1576         // Shouldn't really be possible
1577         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Invalid CIB source");
1578         return clean_up(CRM_EX_ERROR);
1579     }
1580 
1581     reconcile_output_format(args);
1582     add_output_args();
1583 
1584     if (args->version && output_format == mon_output_console) {
1585         /* Use the text output format here if we are in curses mode but were given
1586          * --version.  Displaying version information uses printf, and then we
1587          *  immediately exit.  We don't want to initialize curses for that.
1588          */
1589         rc = pcmk__output_new(&out, "text", args->output_dest, argv);
1590     } else {
1591         rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
1592     }
1593 
1594     if (rc != pcmk_rc_ok) {
1595         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error creating output format %s: %s",
1596                     args->output_ty, pcmk_rc_str(rc));
1597         return clean_up(CRM_EX_ERROR);
1598     }
1599 
1600     /* output_format MUST NOT BE CHANGED AFTER THIS POINT. */
1601 
1602     /* Apply --include/--exclude flags we used internally.  There's no error reporting
1603      * here because this would be a programming error.
1604      */
1605     apply_include_exclude(options.includes_excludes, output_format, &error);
1606 
1607     /* And now apply any --include/--exclude flags the user gave on the command line.
1608      * These are done in a separate pass from the internal ones because we want to
1609      * make sure whatever the user specifies overrides whatever we do.
1610      */
1611     if (!apply_include_exclude(options.user_includes_excludes, output_format, &error)) {
1612         return clean_up(CRM_EX_USAGE);
1613     }
1614 
1615     /* Sync up the initial value of interactive_fence_level with whatever was set with
1616      * --include/--exclude= options.
1617      */
1618     if (pcmk_is_set(show, mon_show_fencing_all)) {
1619         interactive_fence_level = 3;
1620     } else if (pcmk_is_set(show, mon_show_fence_worked)) {
1621         interactive_fence_level = 2;
1622     } else if (pcmk_any_flags_set(show, mon_show_fence_failed | mon_show_fence_pending)) {
1623         interactive_fence_level = 1;
1624     } else {
1625         interactive_fence_level = 0;
1626     }
1627 
1628     crm_mon_register_messages(out);
1629     pe__register_messages(out);
1630     stonith__register_messages(out);
1631 
1632     if (args->version) {
1633         out->version(out, false);
1634         return clean_up(CRM_EX_OK);
1635     }
1636 
1637     /* Extra sanity checks when in CGI mode */
1638     if (output_format == mon_output_cgi) {
1639         if (cib->variant == cib_file) {
1640             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode used with CIB file");
1641             return clean_up(CRM_EX_USAGE);
1642         } else if (options.external_agent != NULL) {
1643             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with --external-agent");
1644             return clean_up(CRM_EX_USAGE);
1645         } else if (options.daemonize == TRUE) {
1646             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with -d");
1647             return clean_up(CRM_EX_USAGE);
1648         }
1649     }
1650 
1651     if (output_format == mon_output_xml || output_format == mon_output_legacy_xml) {
1652         options.mon_ops |= mon_op_print_timing | mon_op_inactive_resources;
1653 
1654         if (!options.daemonize) {
1655             options.mon_ops |= mon_op_one_shot;
1656         }
1657     }
1658 
1659     if ((output_format == mon_output_html || output_format == mon_output_cgi) &&
1660         out->dest != stdout) {
1661         pcmk__html_add_header("meta", "http-equiv", "refresh", "content",
1662                               pcmk__itoa(options.reconnect_ms / 1000), NULL);
1663     }
1664 
1665     crm_info("Starting %s", crm_system_name);
1666 
1667     cib__set_output(cib, out);
1668 
1669     if (pcmk_is_set(options.mon_ops, mon_op_one_shot)) {
1670         one_shot();
1671     }
1672 
1673     do {
1674         out->info(out,"Waiting until cluster is available on this node ...");
1675 
1676         rc = pacemakerd_status();
1677         if (rc == pcmk_rc_ok) {
1678             fencing_connect();
1679             rc = cib_connect(TRUE);
1680         }
1681 
1682         if (rc != pcmk_rc_ok) {
1683             pcmk__sleep_ms(options.reconnect_ms);
1684 #if CURSES_ENABLED
1685             if (output_format == mon_output_console) {
1686                 clear();
1687                 refresh();
1688             }
1689 #endif
1690         } else if (output_format == mon_output_html && out->dest != stdout) {
1691             printf("Writing html to %s ...\n", args->output_dest);
1692         }
1693 
1694     } while (rc == ENOTCONN);
1695 
1696     handle_connection_failures(rc);
1697     set_fencing_options(interactive_fence_level);
1698     mon_refresh_display(NULL);
1699 
1700     mainloop = g_main_loop_new(NULL, FALSE);
1701 
1702     mainloop_add_signal(SIGTERM, mon_shutdown);
1703     mainloop_add_signal(SIGINT, mon_shutdown);
1704 #if CURSES_ENABLED
1705     if (output_format == mon_output_console) {
1706         ncurses_winch_handler = crm_signal_handler(SIGWINCH, mon_winresize);
1707         if (ncurses_winch_handler == SIG_DFL ||
1708             ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
1709             ncurses_winch_handler = NULL;
1710 
1711         io_channel = g_io_channel_unix_new(STDIN_FILENO);
1712         g_io_add_watch(io_channel, G_IO_IN, detect_user_input, NULL);
1713     }
1714 #endif
1715 
1716     /* When refresh_trigger->trigger is set to TRUE, call mon_refresh_display.  In
1717      * this file, that is anywhere mainloop_set_trigger is called.
1718      */
1719     refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
1720 
1721     g_main_loop_run(mainloop);
1722     g_main_loop_unref(mainloop);
1723 
1724     if (io_channel != NULL) {
1725         g_io_channel_shutdown(io_channel, TRUE, NULL);
1726     }
1727 
1728     crm_info("Exiting %s", crm_system_name);
1729 
1730     return clean_up(CRM_EX_OK);
1731 }
1732 
1733 /*!
1734  * \internal
1735  * \brief Print one-line status suitable for use with monitoring software
1736  *
1737  * \param[in] data_set  Working set of CIB state
1738  *
1739  * \note This function's output (and the return code when the program exits)
1740  *       should conform to https://www.monitoring-plugins.org/doc/guidelines.html
1741  */
1742 static void
1743 print_simple_status(pcmk__output_t *out, pe_working_set_t * data_set,
     /*  */
1744                     unsigned int mon_ops)
1745 {
1746     GList *gIter = NULL;
1747     int nodes_online = 0;
1748     int nodes_standby = 0;
1749     int nodes_maintenance = 0;
1750     char *offline_nodes = NULL;
1751     size_t offline_nodes_len = 0;
1752     gboolean no_dc = FALSE;
1753     gboolean offline = FALSE;
1754 
1755     if (data_set->dc_node == NULL) {
1756         mon_ops |= mon_op_has_warnings;
1757         no_dc = TRUE;
1758     }
1759 
1760     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1761         pe_node_t *node = (pe_node_t *) gIter->data;
1762 
1763         if (node->details->standby && node->details->online) {
1764             nodes_standby++;
1765         } else if (node->details->maintenance && node->details->online) {
1766             nodes_maintenance++;
1767         } else if (node->details->online) {
1768             nodes_online++;
1769         } else {
1770             char *s = crm_strdup_printf("offline node: %s", node->details->uname);
1771             /* coverity[leaked_storage] False positive */
1772             pcmk__add_word(&offline_nodes, &offline_nodes_len, s);
1773             free(s);
1774             mon_ops |= mon_op_has_warnings;
1775             offline = TRUE;
1776         }
1777     }
1778 
1779     if (pcmk_is_set(mon_ops, mon_op_has_warnings)) {
1780         out->info(out, "CLUSTER WARN: %s%s%s",
1781                   no_dc ? "No DC" : "",
1782                   no_dc && offline ? ", " : "",
1783                   (offline? offline_nodes : ""));
1784         free(offline_nodes);
1785     } else {
1786         char *nodes_standby_s = NULL;
1787         char *nodes_maint_s = NULL;
1788 
1789         if (nodes_standby > 0) {
1790             nodes_standby_s = crm_strdup_printf(", %d standby node%s", nodes_standby,
1791                                                 pcmk__plural_s(nodes_standby));
1792         }
1793 
1794         if (nodes_maintenance > 0) {
1795             nodes_maint_s = crm_strdup_printf(", %d maintenance node%s",
1796                                               nodes_maintenance,
1797                                               pcmk__plural_s(nodes_maintenance));
1798         }
1799 
1800         out->info(out, "CLUSTER OK: %d node%s online%s%s, "
1801                        "%d resource instance%s configured",
1802                   nodes_online, pcmk__plural_s(nodes_online),
1803                   nodes_standby_s != NULL ? nodes_standby_s : "",
1804                   nodes_maint_s != NULL ? nodes_maint_s : "",
1805                   data_set->ninstances, pcmk__plural_s(data_set->ninstances));
1806 
1807         free(nodes_standby_s);
1808         free(nodes_maint_s);
1809     }
1810     /* coverity[leaked_storage] False positive */
1811 }
1812 
1813 static int
1814 send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
     /*  */
1815                  int status, const char *desc)
1816 {
1817     pid_t pid;
1818 
1819     /*setenv needs chars, these are ints */
1820     char *rc_s = pcmk__itoa(rc);
1821     char *status_s = pcmk__itoa(status);
1822     char *target_rc_s = pcmk__itoa(target_rc);
1823 
1824     crm_debug("Sending external notification to '%s' via '%s'", options.external_recipient, options.external_agent);
1825 
1826     if(rsc) {
1827         setenv("CRM_notify_rsc", rsc, 1);
1828     }
1829     if (options.external_recipient) {
1830         setenv("CRM_notify_recipient", options.external_recipient, 1);
1831     }
1832     setenv("CRM_notify_node", node, 1);
1833     setenv("CRM_notify_task", task, 1);
1834     setenv("CRM_notify_desc", desc, 1);
1835     setenv("CRM_notify_rc", rc_s, 1);
1836     setenv("CRM_notify_target_rc", target_rc_s, 1);
1837     setenv("CRM_notify_status", status_s, 1);
1838 
1839     pid = fork();
1840     if (pid == -1) {
1841         crm_perror(LOG_ERR, "notification fork() failed.");
1842     }
1843     if (pid == 0) {
1844         /* crm_debug("notification: I am the child. Executing the nofitication program."); */
1845         execl(options.external_agent, options.external_agent, NULL);
1846         exit(CRM_EX_ERROR);
1847     }
1848 
1849     crm_trace("Finished running custom notification program '%s'.", options.external_agent);
1850     free(target_rc_s);
1851     free(status_s);
1852     free(rc_s);
1853     return 0;
1854 }
1855 
1856 static void
1857 handle_rsc_op(xmlNode * xml, const char *node_id)
     /*  */
1858 {
1859     int rc = -1;
1860     int status = -1;
1861     int target_rc = -1;
1862     gboolean notify = TRUE;
1863 
1864     char *rsc = NULL;
1865     char *task = NULL;
1866     const char *desc = NULL;
1867     const char *magic = NULL;
1868     const char *id = NULL;
1869     const char *node = NULL;
1870 
1871     xmlNode *n = xml;
1872     xmlNode * rsc_op = xml;
1873 
1874     if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
1875         xmlNode *cIter;
1876 
1877         for(cIter = xml->children; cIter; cIter = cIter->next) {
1878             handle_rsc_op(cIter, node_id);
1879         }
1880 
1881         return;
1882     }
1883 
1884     id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
1885     if (id == NULL) {
1886         /* Compatibility with <= 1.1.5 */
1887         id = ID(rsc_op);
1888     }
1889 
1890     magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
1891     if (magic == NULL) {
1892         /* non-change */
1893         return;
1894     }
1895 
1896     if (!decode_transition_magic(magic, NULL, NULL, NULL, &status, &rc,
1897                                  &target_rc)) {
1898         crm_err("Invalid event %s detected for %s", magic, id);
1899         return;
1900     }
1901 
1902     if (parse_op_key(id, &rsc, &task, NULL) == FALSE) {
1903         crm_err("Invalid event detected for %s", id);
1904         goto bail;
1905     }
1906 
1907     node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
1908 
1909     while (n != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(n), pcmk__str_casei)) {
1910         n = n->parent;
1911     }
1912 
1913     if(node == NULL && n) {
1914         node = crm_element_value(n, XML_ATTR_UNAME);
1915     }
1916 
1917     if (node == NULL && n) {
1918         node = ID(n);
1919     }
1920 
1921     if (node == NULL) {
1922         node = node_id;
1923     }
1924 
1925     if (node == NULL) {
1926         crm_err("No node detected for event %s (%s)", magic, id);
1927         goto bail;
1928     }
1929 
1930     /* look up where we expected it to be? */
1931     desc = pcmk_strerror(pcmk_ok);
1932     if (status == PCMK_LRM_OP_DONE && target_rc == rc) {
1933         crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
1934         if (rc == PCMK_OCF_NOT_RUNNING) {
1935             notify = FALSE;
1936         }
1937 
1938     } else if (status == PCMK_LRM_OP_DONE) {
1939         desc = services_ocf_exitcode_str(rc);
1940         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1941 
1942     } else {
1943         desc = services_lrm_status_str(status);
1944         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1945     }
1946 
1947     if (notify && options.external_agent) {
1948         send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
1949     }
1950   bail:
1951     free(rsc);
1952     free(task);
1953 }
1954 
1955 /* This function is just a wrapper around mainloop_set_trigger so that it can be
1956  * called from a mainloop directly.  It's simply another way of ensuring the screen
1957  * gets redrawn.
1958  */
1959 static gboolean
1960 mon_trigger_refresh(gpointer user_data)
     /*  */
1961 {
1962     mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
1963     return FALSE;
1964 }
1965 
1966 static void
1967 crm_diff_update_v2(const char *event, xmlNode * msg)
     /*  */
1968 {
1969     xmlNode *change = NULL;
1970     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
1971 
1972     for (change = pcmk__xml_first_child(diff); change != NULL;
1973          change = pcmk__xml_next(change)) {
1974         const char *name = NULL;
1975         const char *op = crm_element_value(change, XML_DIFF_OP);
1976         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
1977         xmlNode *match = NULL;
1978         const char *node = NULL;
1979 
1980         if(op == NULL) {
1981             continue;
1982 
1983         } else if(strcmp(op, "create") == 0) {
1984             match = change->children;
1985 
1986         } else if(strcmp(op, "move") == 0) {
1987             continue;
1988 
1989         } else if(strcmp(op, "delete") == 0) {
1990             continue;
1991 
1992         } else if(strcmp(op, "modify") == 0) {
1993             match = first_named_child(change, XML_DIFF_RESULT);
1994             if(match) {
1995                 match = match->children;
1996             }
1997         }
1998 
1999         if(match) {
2000             name = (const char *)match->name;
2001         }
2002 
2003         crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
2004         if(xpath == NULL) {
2005             /* Version field, ignore */
2006 
2007         } else if(name == NULL) {
2008             crm_debug("No result for %s operation to %s", op, xpath);
2009             CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
2010 
2011         } else if(strcmp(name, XML_TAG_CIB) == 0) {
2012             xmlNode *state = NULL;
2013             xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
2014 
2015             for (state = pcmk__xe_first_child(status); state != NULL;
2016                  state = pcmk__xe_next(state)) {
2017 
2018                 node = crm_element_value(state, XML_ATTR_UNAME);
2019                 if (node == NULL) {
2020                     node = ID(state);
2021                 }
2022                 handle_rsc_op(state, node);
2023             }
2024 
2025         } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
2026             xmlNode *state = NULL;
2027 
2028             for (state = pcmk__xe_first_child(match); state != NULL;
2029                  state = pcmk__xe_next(state)) {
2030 
2031                 node = crm_element_value(state, XML_ATTR_UNAME);
2032                 if (node == NULL) {
2033                     node = ID(state);
2034                 }
2035                 handle_rsc_op(state, node);
2036             }
2037 
2038         } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
2039             node = crm_element_value(match, XML_ATTR_UNAME);
2040             if (node == NULL) {
2041                 node = ID(match);
2042             }
2043             handle_rsc_op(match, node);
2044 
2045         } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
2046             node = ID(match);
2047             handle_rsc_op(match, node);
2048 
2049         } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
2050             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2051 
2052             handle_rsc_op(match, local_node);
2053             free(local_node);
2054 
2055         } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
2056             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2057 
2058             handle_rsc_op(match, local_node);
2059             free(local_node);
2060 
2061         } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
2062             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
2063 
2064             handle_rsc_op(match, local_node);
2065             free(local_node);
2066 
2067         } else {
2068             crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
2069         }
2070     }
2071 }
2072 
2073 static void
2074 crm_diff_update_v1(const char *event, xmlNode * msg)
     /*  */
2075 {
2076     /* Process operation updates */
2077     xmlXPathObject *xpathObj = xpath_search(msg,
2078                                             "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
2079                                             "//" XML_LRM_TAG_RSC_OP);
2080     int lpc = 0, max = numXpathResults(xpathObj);
2081 
2082     for (lpc = 0; lpc < max; lpc++) {
2083         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
2084 
2085         handle_rsc_op(rsc_op, NULL);
2086     }
2087     freeXpathObject(xpathObj);
2088 }
2089 
2090 static void
2091 crm_diff_update(const char *event, xmlNode * msg)
     /*  */
2092 {
2093     int rc = -1;
2094     static bool stale = FALSE;
2095     gboolean cib_updated = FALSE;
2096     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
2097 
2098     out->progress(out, false);
2099 
2100     if (current_cib != NULL) {
2101         rc = xml_apply_patchset(current_cib, diff, TRUE);
2102 
2103         switch (rc) {
2104             case -pcmk_err_diff_resync:
2105             case -pcmk_err_diff_failed:
2106                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
2107                 free_xml(current_cib); current_cib = NULL;
2108                 break;
2109             case pcmk_ok:
2110                 cib_updated = TRUE;
2111                 break;
2112             default:
2113                 crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
2114                 free_xml(current_cib); current_cib = NULL;
2115         }
2116     }
2117 
2118     if (current_cib == NULL) {
2119         crm_trace("Re-requesting the full cib");
2120         cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
2121     }
2122 
2123     if (options.external_agent) {
2124         int format = 0;
2125         crm_element_value_int(diff, "format", &format);
2126         switch(format) {
2127             case 1:
2128                 crm_diff_update_v1(event, msg);
2129                 break;
2130             case 2:
2131                 crm_diff_update_v2(event, msg);
2132                 break;
2133             default:
2134                 crm_err("Unknown patch format: %d", format);
2135         }
2136     }
2137 
2138     if (current_cib == NULL) {
2139         if(!stale) {
2140             out->info(out, "--- Stale data ---");
2141         }
2142         stale = TRUE;
2143         return;
2144     }
2145 
2146     stale = FALSE;
2147     refresh_after_event(cib_updated, FALSE);
2148 }
2149 
2150 static int
2151 get_fencing_history(stonith_history_t **stonith_history)
     /*  */
2152 {
2153     int rc = 0;
2154 
2155     while (pcmk_is_set(options.mon_ops, mon_op_fence_history)) {
2156         if (st != NULL) {
2157             rc = st->cmds->history(st, st_opt_sync_call, NULL, stonith_history, 120);
2158 
2159             if (rc == 0) {
2160                 *stonith_history = stonith__sort_history(*stonith_history);
2161                 if (!pcmk_is_set(options.mon_ops, mon_op_fence_full_history)
2162                     && (output_format != mon_output_xml)) {
2163 
2164                     *stonith_history = pcmk__reduce_fence_history(*stonith_history);
2165                 }
2166                 break; /* all other cases are errors */
2167             }
2168         } else {
2169             rc = ENOTCONN;
2170             break;
2171         }
2172     }
2173 
2174     return rc;
2175 }
2176 
2177 static int
2178 mon_refresh_display(gpointer user_data)
     /*  */
2179 {
2180     xmlNode *cib_copy = copy_xml(current_cib);
2181     stonith_history_t *stonith_history = NULL;
2182     int history_rc = 0;
2183     GList *unames = NULL;
2184     GList *resources = NULL;
2185 
2186     last_refresh = time(NULL);
2187 
2188     if (cli_config_update(&cib_copy, NULL, FALSE) == FALSE) {
2189         clean_up_cib_connection();
2190         out->err(out, "Upgrade failed: %s", pcmk_strerror(-pcmk_err_schema_validation));
2191         clean_up(CRM_EX_CONFIG);
2192         return 0;
2193     }
2194 
2195     /* get the stonith-history if there is evidence we need it */
2196     history_rc = get_fencing_history(&stonith_history);
2197 
2198     if (mon_data_set == NULL) {
2199         mon_data_set = pe_new_working_set();
2200         CRM_ASSERT(mon_data_set != NULL);
2201     }
2202     pe__set_working_set_flags(mon_data_set, pe_flag_no_compat);
2203 
2204     mon_data_set->input = cib_copy;
2205     mon_data_set->priv = out;
2206     cluster_status(mon_data_set);
2207 
2208     /* Unpack constraints if any section will need them
2209      * (tickets may be referenced in constraints but not granted yet,
2210      * and bans need negative location constraints) */
2211     if (pcmk_is_set(show, mon_show_bans) || pcmk_is_set(show, mon_show_tickets)) {
2212         xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS,
2213                                                    mon_data_set->input);
2214         unpack_constraints(cib_constraints, mon_data_set);
2215     }
2216 
2217     if (options.daemonize) {
2218         out->reset(out);
2219     }
2220 
2221     unames = pe__build_node_name_list(mon_data_set, options.only_node);
2222     resources = pe__build_rsc_list(mon_data_set, options.only_rsc);
2223 
2224     switch (output_format) {
2225         case mon_output_html:
2226         case mon_output_cgi:
2227             if (print_html_status(mon_data_set, crm_errno2exit(history_rc),
2228                                   stonith_history, options.mon_ops,
2229                                   get_resource_display_options(options.mon_ops),
2230                                   show, options.neg_location_prefix,
2231                                   unames, resources) != 0) {
2232                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_CANTCREAT, "Critical: Unable to output html file");
2233                 clean_up(CRM_EX_CANTCREAT);
2234                 return 0;
2235             }
2236             break;
2237 
2238         case mon_output_legacy_xml:
2239         case mon_output_xml:
2240             print_xml_status(mon_data_set, crm_errno2exit(history_rc),
2241                              stonith_history, options.mon_ops,
2242                              get_resource_display_options(options.mon_ops), show,
2243                              options.neg_location_prefix, unames, resources);
2244             break;
2245 
2246         case mon_output_monitor:
2247             print_simple_status(out, mon_data_set, options.mon_ops);
2248             if (pcmk_is_set(options.mon_ops, mon_op_has_warnings)) {
2249                 clean_up(MON_STATUS_WARN);
2250                 return FALSE;
2251             }
2252             break;
2253 
2254         case mon_output_console:
2255             /* If curses is not enabled, this will just fall through to the plain
2256              * text case.
2257              */
2258 #if CURSES_ENABLED
2259             blank_screen();
2260             print_status(mon_data_set, crm_errno2exit(history_rc), stonith_history,
2261                          options.mon_ops, get_resource_display_options(options.mon_ops),
2262                          show, options.neg_location_prefix, unames, resources);
2263             refresh();
2264             break;
2265 #endif
2266 
2267         case mon_output_plain:
2268             print_status(mon_data_set, crm_errno2exit(history_rc), stonith_history,
2269                          options.mon_ops, get_resource_display_options(options.mon_ops),
2270                          show, options.neg_location_prefix, unames, resources);
2271             break;
2272 
2273         case mon_output_unset:
2274         case mon_output_none:
2275             break;
2276     }
2277 
2278     if (options.daemonize) {
2279         out->finish(out, CRM_EX_OK, true, NULL);
2280     }
2281 
2282     g_list_free_full(unames, free);
2283     g_list_free_full(resources, free);
2284 
2285     stonith_history_free(stonith_history);
2286     stonith_history = NULL;
2287     pe_reset_working_set(mon_data_set);
2288     return 1;
2289 }
2290 
2291 /* This function is called for fencing events (see fencing_connect for which ones) when
2292  * --watch-fencing is used on the command line.
2293  */
2294 static void
2295 mon_st_callback_event(stonith_t * st, stonith_event_t * e)
     /*  */
2296 {
2297     if (st->state == stonith_disconnected) {
2298         /* disconnect cib as well and have everything reconnect */
2299         mon_cib_connection_destroy(NULL);
2300     } else if (options.external_agent) {
2301         char *desc = crm_strdup_printf("Operation %s requested by %s for peer %s: %s (ref=%s)",
2302                                     e->operation, e->origin, e->target, pcmk_strerror(e->result),
2303                                     e->id);
2304         send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
2305         free(desc);
2306     }
2307 }
2308 
2309 /* Cause the screen to be redrawn (via mainloop_set_trigger) when various conditions are met:
2310  *
2311  * - If the last update occurred more than reconnect_ms ago (defaults to 5s, but
2312  *   can be changed via the -i command line option), or
2313  * - After every 10 CIB updates, or
2314  * - If it's been 2s since the last update
2315  *
2316  * This function sounds like it would be more broadly useful, but it is only called when a
2317  * fencing event is received or a CIB diff occurrs.
2318  */
2319 static void
2320 refresh_after_event(gboolean data_updated, gboolean enforce)
     /*  */
2321 {
2322     static int updates = 0;
2323     time_t now = time(NULL);
2324 
2325     if (data_updated) {
2326         updates++;
2327     }
2328 
2329     if(refresh_timer == NULL) {
2330         refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
2331     }
2332 
2333     if (reconnect_timer > 0) {
2334         /* we will receive a refresh request after successful reconnect */
2335         mainloop_timer_stop(refresh_timer);
2336         return;
2337     }
2338 
2339     /* as we're not handling initial failure of fencer-connection as
2340      * fatal give it a retry here
2341      * not getting here if cib-reconnection is already on the way
2342      */
2343     fencing_connect();
2344 
2345     if (enforce ||
2346         ((now - last_refresh) > (options.reconnect_ms / 1000)) ||
2347         updates >= 10) {
2348         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
2349         mainloop_timer_stop(refresh_timer);
2350         updates = 0;
2351 
2352     } else {
2353         mainloop_timer_start(refresh_timer);
2354     }
2355 }
2356 
2357 /* This function is called for fencing events (see fencing_connect for which ones) when
2358  * --watch-fencing is NOT used on the command line.
2359  */
2360 static void
2361 mon_st_callback_display(stonith_t * st, stonith_event_t * e)
     /*  */
2362 {
2363     if (st->state == stonith_disconnected) {
2364         /* disconnect cib as well and have everything reconnect */
2365         mon_cib_connection_destroy(NULL);
2366     } else {
2367         out->progress(out, false);
2368         refresh_after_event(TRUE, FALSE);
2369     }
2370 }
2371 
2372 static void
2373 clean_up_cib_connection(void)
     /*  */
2374 {
2375     if (cib == NULL) {
2376         return;
2377     }
2378 
2379     cib->cmds->signoff(cib);
2380     cib_delete(cib);
2381     cib = NULL;
2382 }
2383 
2384 static void
2385 clean_up_fencing_connection(void)
     /*  */
2386 {
2387     if (st == NULL) {
2388         return;
2389     }
2390 
2391     if (st->state != stonith_disconnected) {
2392         st->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT);
2393         st->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE);
2394         st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY);
2395         st->cmds->disconnect(st);
2396     }
2397 
2398     stonith_api_delete(st);
2399     st = NULL;
2400 }
2401 
2402 /*
2403  * De-init ncurses, disconnect from the CIB manager, disconnect fencing,
2404  * deallocate memory and show usage-message if requested.
2405  *
2406  * We don't actually return, but nominally returning crm_exit_t allows a usage
2407  * like "return clean_up(exit_code);" which helps static analysis understand the
2408  * code flow.
2409  */
2410 static crm_exit_t
2411 clean_up(crm_exit_t exit_code)
     /*  */
2412 {
2413     /* Quitting crm_mon is much more complicated than it ought to be. */
2414 
2415     /* (1) Close connections, free things, etc. */
2416     clean_up_cib_connection();
2417     clean_up_fencing_connection();
2418     free(options.neg_location_prefix);
2419     free(options.only_node);
2420     free(options.only_rsc);
2421     free(options.pid_file);
2422     g_slist_free_full(options.includes_excludes, free);
2423 
2424     pe_free_working_set(mon_data_set);
2425     mon_data_set = NULL;
2426 
2427     g_strfreev(processed_args);
2428 
2429     /* (2) If this is abnormal termination and we're in curses mode, shut down
2430      * curses first.  Any messages displayed to the screen before curses is shut
2431      * down will be lost because doing the shut down will also restore the
2432      * screen to whatever it looked like before crm_mon was started.
2433      */
2434     if ((error != NULL || exit_code == CRM_EX_USAGE) && output_format == mon_output_console) {
2435         out->finish(out, exit_code, false, NULL);
2436         pcmk__output_free(out);
2437         out = NULL;
2438     }
2439 
2440     /* (3) If this is a command line usage related failure, print the usage
2441      * message.
2442      */
2443     if (exit_code == CRM_EX_USAGE && (output_format == mon_output_console || output_format == mon_output_plain)) {
2444         char *help = g_option_context_get_help(context, TRUE, NULL);
2445 
2446         fprintf(stderr, "%s", help);
2447         g_free(help);
2448     }
2449 
2450     pcmk__free_arg_context(context);
2451 
2452     /* (4) If this is any kind of error, print the error out and exit.  Make
2453      * sure to handle situations both before and after formatted output is
2454      * set up.  We want errors to appear formatted if at all possible.
2455      */
2456     if (error != NULL) {
2457         if (out != NULL) {
2458             out->err(out, "%s: %s", g_get_prgname(), error->message);
2459             out->finish(out, exit_code, true, NULL);
2460             pcmk__output_free(out);
2461         } else {
2462             fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
2463         }
2464 
2465         g_clear_error(&error);
2466         crm_exit(exit_code);
2467     }
2468 
2469     /* (5) Print formatted output to the screen if we made it far enough in
2470      * crm_mon to be able to do so.
2471      */
2472     if (out != NULL) {
2473         if (!options.daemonize) {
2474             out->finish(out, exit_code, true, NULL);
2475         }
2476 
2477         pcmk__output_free(out);
2478         pcmk__unregister_formats();
2479     }
2480 
2481     crm_exit(exit_code);
2482 }