daemons/fenced/fenced

/* */
This source file includes following definitions.
stonith_send_broadcast_history
stonith_remove_history_entry
stonith_fence_history_cleanup
op_time_sort
stonith_fence_history_trim
stonith_xml_history_to_list
stonith_local_history_diff_and_merge
stonith_local_history
stonith_fence_history
   1 /*
   2  * Copyright 2009-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdio.h>
  13 #include <unistd.h>
  14 #include <stdlib.h>
  15 
  16 #include <crm/crm.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/common/ipc.h>
  19 #include <crm/common/ipc_internal.h>
  20 #include <crm/cluster/internal.h>
  21 
  22 #include <crm/stonith-ng.h>
  23 #include <crm/fencing/internal.h>
  24 #include <crm/common/xml.h>
  25 #include <crm/common/xml_internal.h>
  26 
  27 #include <pacemaker-fenced.h>
  28 
  29 #define MAX_STONITH_HISTORY 500
  30 
  31 /*!
  32  * \internal
  33  * \brief Send a broadcast to all nodes to trigger cleanup or
  34  *        history synchronisation
  35  *
  36  * \param[in] history   Optional history to be attached
  37  * \param[in] callopts  We control cleanup via a flag in the callopts
  38  * \param[in] target    Cleanup can be limited to certain fence-targets
  39  */
  40 static void
  41 stonith_send_broadcast_history(xmlNode *history,
     /*  */
  42                                int callopts,
  43                                const char *target)
  44 {
  45     xmlNode *bcast = create_xml_node(NULL, "stonith_command");
  46     xmlNode *data = create_xml_node(NULL, __func__);
  47 
  48     if (target) {
  49         crm_xml_add(data, F_STONITH_TARGET, target);
  50     }
  51     crm_xml_add(bcast, F_TYPE, T_STONITH_NG);
  52     crm_xml_add(bcast, F_SUBTYPE, "broadcast");
  53     crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY);
  54     crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts);
  55     if (history) {
  56         add_node_copy(data, history);
  57     }
  58     add_message_xml(bcast, F_STONITH_CALLDATA, data);
  59     send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
  60 
  61     free_xml(data);
  62     free_xml(bcast);
  63 }
  64 
  65 static gboolean
  66 stonith_remove_history_entry (gpointer key,
     /*  */
  67                               gpointer value,
  68                               gpointer user_data)
  69 {
  70     remote_fencing_op_t *op = value;
  71     const char *target = (const char *) user_data;
  72 
  73     if ((op->state == st_failed) || (op->state == st_done)) {
  74         if ((target) && (strcmp(op->target, target) != 0)) {
  75             return FALSE;
  76         }
  77         return TRUE;
  78     }
  79 
  80     return FALSE; /* don't clean pending operations */
  81 }
  82 
  83 /*!
  84  * \internal
  85  * \brief Send out a cleanup broadcast or do a local history-cleanup
  86  *
  87  * \param[in] target    Cleanup can be limited to certain fence-targets
  88  * \param[in] broadcast Send out a cleanup broadcast
  89  */
  90 static void
  91 stonith_fence_history_cleanup(const char *target,
     /*  */
  92                               gboolean broadcast)
  93 {
  94     if (broadcast) {
  95         stonith_send_broadcast_history(NULL,
  96                                        st_opt_cleanup | st_opt_discard_reply,
  97                                        target);
  98         /* we'll do the local clean when we receive back our own broadcast */
  99     } else if (stonith_remote_op_list) {
 100         g_hash_table_foreach_remove(stonith_remote_op_list,
 101                              stonith_remove_history_entry,
 102                              (gpointer) target);
 103         fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
 104     }
 105 }
 106 
 107 /* keeping the length of fence-history within bounds
 108  * =================================================
 109  *
 110  * If things are really running wild a lot of fencing-attempts
 111  * might fill up the hash-map, eventually using up a lot
 112  * of memory and creating huge history-sync messages.
 113  * Before the history being synced across nodes at least
 114  * the reboot of a cluster-node helped keeping the
 115  * history within bounds even though not in a reliable
 116  * manner.
 117  *
 118  * stonith_remote_op_list isn't sorted for time-stamps
 119  * thus it would be kind of expensive to delete e.g.
 120  * the oldest entry if it would grow past MAX_STONITH_HISTORY
 121  * entries.
 122  * It is more efficient to purge MAX_STONITH_HISTORY/2
 123  * entries whenever the list grows beyond MAX_STONITH_HISTORY.
 124  * (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
 125  * That done on a per-node-base might raise the
 126  * probability of large syncs to occur.
 127  * Things like introducing a broadcast to purge
 128  * MAX_STONITH_HISTORY/2 entries or not sync above a certain
 129  * threshold coming to mind ...
 130  * Simplest thing though is to purge the full history
 131  * throughout the cluster once MAX_STONITH_HISTORY is reached.
 132  * On the other hand this leads to purging the history in
 133  * situations where it would be handy to have it probably.
 134  */
 135 
 136 
 137 static int
 138 op_time_sort(const void *a_voidp, const void *b_voidp)
     /*  */
 139 {
 140     const remote_fencing_op_t **a = (const remote_fencing_op_t **) a_voidp;
 141     const remote_fencing_op_t **b = (const remote_fencing_op_t **) b_voidp;
 142     gboolean a_pending = ((*a)->state != st_failed) && ((*a)->state != st_done);
 143     gboolean b_pending = ((*b)->state != st_failed) && ((*b)->state != st_done);
 144 
 145     if (a_pending && b_pending) {
 146         return 0;
 147     } else if (a_pending) {
 148         return -1;
 149     } else if (b_pending) {
 150         return 1;
 151     } else if ((*b)->completed == (*a)->completed) {
 152         if ((*b)->completed_nsec > (*a)->completed_nsec) {
 153             return 1;
 154         } else if ((*b)->completed_nsec == (*a)->completed_nsec) {
 155             return 0;
 156         }
 157     } else if ((*b)->completed > (*a)->completed) {
 158         return 1;
 159     }
 160 
 161     return -1;
 162 }
 163 
 164 
 165 /*!
 166  * \internal
 167  * \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
 168  *        once over MAX_STONITH_HISTORY
 169  */
 170 void
 171 stonith_fence_history_trim(void)
     /*  */
 172 {
 173     guint num_ops;
 174 
 175     if (!stonith_remote_op_list) {
 176         return;
 177     }
 178     num_ops = g_hash_table_size(stonith_remote_op_list);
 179     if (num_ops > MAX_STONITH_HISTORY) {
 180         remote_fencing_op_t *ops[num_ops];
 181         remote_fencing_op_t *op = NULL;
 182         GHashTableIter iter;
 183         int i;
 184 
 185         crm_trace("Fencing History growing beyond limit of %d so purge "
 186                   "half of failed/successful attempts", MAX_STONITH_HISTORY);
 187 
 188         /* write all ops into an array */
 189         i = 0;
 190         g_hash_table_iter_init(&iter, stonith_remote_op_list);
 191         while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
 192             ops[i++] = op;
 193         }
 194         /* run quicksort over the array so that we get pending ops
 195          * first and then sorted most recent to oldest
 196          */
 197         qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort);
 198         /* purgest oldest half of the history entries */
 199         for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) {
 200             /* keep pending ops even if they shouldn't fill more than
 201              * half of our buffer
 202              */
 203             if ((ops[i]->state == st_failed) || (ops[i]->state == st_done)) {
 204                 g_hash_table_remove(stonith_remote_op_list, ops[i]->id);
 205             }
 206         }
 207         /* we've just purged valid data from the list so there is no need
 208          * to create a notification - if displayed it can stay
 209          */
 210     }
 211 }
 212 
 213 /*!
 214  * \internal
 215  * \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
 216  *
 217  * \param[in] history   Fence-history in xml
 218  *
 219  * \return Fence-history as hash-table
 220  */
 221 static GHashTable *
 222 stonith_xml_history_to_list(const xmlNode *history)
     /*  */
 223 {
 224     xmlNode *xml_op = NULL;
 225     GHashTable *rv = NULL;
 226 
 227     init_stonith_remote_op_hash_table(&rv);
 228 
 229     CRM_LOG_ASSERT(rv != NULL);
 230 
 231     for (xml_op = pcmk__xml_first_child(history); xml_op != NULL;
 232          xml_op = pcmk__xml_next(xml_op)) {
 233         remote_fencing_op_t *op = NULL;
 234         char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID);
 235         int state;
 236         int exit_status = CRM_EX_OK;
 237         int execution_status = PCMK_EXEC_DONE;
 238         long long completed;
 239         long long completed_nsec = 0L;
 240 
 241         if (!id) {
 242             crm_warn("Malformed fencing history received from peer");
 243             continue;
 244         }
 245 
 246         crm_trace("Attaching op %s to hashtable", id);
 247 
 248         op = calloc(1, sizeof(remote_fencing_op_t));
 249 
 250         op->id = id;
 251         op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET);
 252         op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION);
 253         op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN);
 254         op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE);
 255         op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME);
 256         crm_element_value_ll(xml_op, F_STONITH_DATE, &completed);
 257         op->completed = (time_t) completed;
 258         crm_element_value_ll(xml_op, F_STONITH_DATE_NSEC, &completed_nsec);
 259         op->completed_nsec = completed_nsec;
 260         crm_element_value_int(xml_op, F_STONITH_STATE, &state);
 261         op->state = (enum op_state) state;
 262 
 263         /* @COMPAT We can't use stonith__xe_get_result() here because
 264          * fencers <2.1.3 didn't include results, leading it to assume an error
 265          * status. Instead, set an unknown status in that case.
 266          */
 267         if ((crm_element_value_int(xml_op, XML_LRM_ATTR_RC, &exit_status) < 0)
 268             || (crm_element_value_int(xml_op, XML_LRM_ATTR_OPSTATUS,
 269                                       &execution_status) < 0)) {
 270             exit_status = CRM_EX_INDETERMINATE;
 271             execution_status = PCMK_EXEC_UNKNOWN;
 272         }
 273         pcmk__set_result(&op->result, exit_status, execution_status,
 274                          crm_element_value(xml_op, XML_LRM_ATTR_EXIT_REASON));
 275         pcmk__set_result_output(&op->result,
 276                                 crm_element_value_copy(xml_op, F_STONITH_OUTPUT),
 277                                 NULL);
 278 
 279 
 280         g_hash_table_replace(rv, id, op);
 281         CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
 282     }
 283 
 284     return rv;
 285 }
 286 
 287 /*!
 288  * \internal
 289  * \brief Craft xml difference between local fence-history and a history
 290  *        coming from remote, and merge the remote history into the local
 291  *
 292  * \param[in,out] remote_history  Fence-history as hash-table (may be NULL)
 293  * \param[in]     add_id          If crafting the answer for an API
 294  *                                history-request there is no need for the id
 295  * \param[in]     target          Optionally limit to certain fence-target
 296  *
 297  * \return The fence-history as xml
 298  */
 299 static xmlNode *
 300 stonith_local_history_diff_and_merge(GHashTable *remote_history,
     /*  */
 301                                      gboolean add_id, const char *target)
 302 {
 303     xmlNode *history = NULL;
 304     GHashTableIter iter;
 305     remote_fencing_op_t *op = NULL;
 306     gboolean updated = FALSE;
 307     int cnt = 0;
 308 
 309     if (stonith_remote_op_list) {
 310             char *id = NULL;
 311 
 312             history = create_xml_node(NULL, F_STONITH_HISTORY_LIST);
 313 
 314             g_hash_table_iter_init(&iter, stonith_remote_op_list);
 315             while (g_hash_table_iter_next(&iter, (void **)&id, (void **)&op)) {
 316                 xmlNode *entry = NULL;
 317 
 318                 if (remote_history) {
 319                     remote_fencing_op_t *remote_op =
 320                         g_hash_table_lookup(remote_history, op->id);
 321 
 322                     if (remote_op) {
 323                         if (stonith__op_state_pending(op->state)
 324                             && !stonith__op_state_pending(remote_op->state)) {
 325 
 326                             crm_debug("Updating outdated pending operation %.8s "
 327                                       "(state=%s) according to the one (state=%s) from "
 328                                       "remote peer history",
 329                                       op->id, stonith_op_state_str(op->state),
 330                                       stonith_op_state_str(remote_op->state));
 331 
 332                             g_hash_table_steal(remote_history, op->id);
 333                             op->id = remote_op->id;
 334                             remote_op->id = id;
 335                             g_hash_table_iter_replace(&iter, remote_op);
 336 
 337                             updated = TRUE;
 338                             continue; /* skip outdated entries */
 339 
 340                         } else if (!stonith__op_state_pending(op->state)
 341                                    && stonith__op_state_pending(remote_op->state)) {
 342 
 343                             crm_debug("Broadcasting operation %.8s (state=%s) to "
 344                                       "update the outdated pending one "
 345                                       "(state=%s) in remote peer history",
 346                                       op->id, stonith_op_state_str(op->state),
 347                                       stonith_op_state_str(remote_op->state));
 348 
 349                             g_hash_table_remove(remote_history, op->id);
 350 
 351                         } else {
 352                             g_hash_table_remove(remote_history, op->id);
 353                             continue; /* skip entries broadcasted already */
 354                         }
 355                     }
 356                 }
 357 
 358                 if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
 359                     continue;
 360                 }
 361 
 362                 cnt++;
 363                 crm_trace("Attaching op %s", op->id);
 364                 entry = create_xml_node(history, STONITH_OP_EXEC);
 365                 if (add_id) {
 366                     crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id);
 367                 }
 368                 crm_xml_add(entry, F_STONITH_TARGET, op->target);
 369                 crm_xml_add(entry, F_STONITH_ACTION, op->action);
 370                 crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
 371                 crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
 372                 crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
 373                 crm_xml_add_ll(entry, F_STONITH_DATE, op->completed);
 374                 crm_xml_add_ll(entry, F_STONITH_DATE_NSEC, op->completed_nsec);
 375                 crm_xml_add_int(entry, F_STONITH_STATE, op->state);
 376                 stonith__xe_set_result(entry, &op->result);
 377             }
 378     }
 379 
 380     if (remote_history) {
 381         init_stonith_remote_op_hash_table(&stonith_remote_op_list);
 382 
 383         updated |= g_hash_table_size(remote_history);
 384 
 385         g_hash_table_iter_init(&iter, remote_history);
 386         while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
 387             if (stonith__op_state_pending(op->state) &&
 388                 pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
 389 
 390                 crm_warn("Failing pending operation %.8s originated by us but "
 391                          "known only from peer history", op->id);
 392                 op->state = st_failed;
 393                 set_fencing_completed(op);
 394 
 395                 /* CRM_EX_EXPIRED + PCMK_EXEC_INVALID prevents finalize_op()
 396                  * from setting a delegate
 397                  */
 398                 pcmk__set_result(&op->result, CRM_EX_EXPIRED, PCMK_EXEC_INVALID,
 399                                  "Initiated by earlier fencer "
 400                                  "process and presumed failed");
 401                 fenced_broadcast_op_result(op, false);
 402             }
 403 
 404             g_hash_table_iter_steal(&iter);
 405             g_hash_table_replace(stonith_remote_op_list, op->id, op);
 406             /* we could trim the history here but if we bail
 407              * out after trim we might miss more recent entries
 408              * of those that might still be in the list
 409              * if we don't bail out trimming once is more
 410              * efficient and memory overhead is minimal as
 411              * we are just moving pointers from one hash to
 412              * another
 413              */
 414         }
 415 
 416         g_hash_table_destroy(remote_history); /* remove what is left */
 417     }
 418 
 419     if (updated) {
 420         stonith_fence_history_trim();
 421         fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
 422     }
 423 
 424     if (cnt == 0) {
 425         free_xml(history);
 426         return NULL;
 427     } else {
 428         return history;
 429     }
 430 }
 431 
 432 /*!
 433  * \internal
 434  * \brief Craft xml from the local fence-history
 435  *
 436  * \param[in] add_id            If crafting the answer for an API
 437  *                              history-request there is no need for the id
 438  * \param[in] target            Optionally limit to certain fence-target
 439  *
 440  * \return The fence-history as xml
 441  */
 442 static xmlNode *
 443 stonith_local_history(gboolean add_id, const char *target)
     /*  */
 444 {
 445     return stonith_local_history_diff_and_merge(NULL, add_id, target);
 446 }
 447 
 448 /*!
 449  * \internal
 450  * \brief Handle fence-history messages (from API or coming in as broadcasts)
 451  *
 452  * \param[in,out] msg          Request XML
 453  * \param[out]    output       Where to set local history, if requested
 454  * \param[in]     remote_peer  If broadcast, peer that sent it
 455  * \param[in]     options      Call options from the request
 456  */
 457 void
 458 stonith_fence_history(xmlNode *msg, xmlNode **output,
     /*  */
 459                       const char *remote_peer, int options)
 460 {
 461     const char *target = NULL;
 462     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER);
 463     xmlNode *out_history = NULL;
 464 
 465     if (dev) {
 466         target = crm_element_value(dev, F_STONITH_TARGET);
 467         if (target && (options & st_opt_cs_nodeid)) {
 468             int nodeid;
 469             crm_node_t *node;
 470 
 471             pcmk__scan_min_int(target, &nodeid, 0);
 472             node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
 473             if (node) {
 474                 target = node->uname;
 475             }
 476         }
 477     }
 478 
 479     if (options & st_opt_cleanup) {
 480         crm_trace("Cleaning up operations on %s in %p", target,
 481                   stonith_remote_op_list);
 482 
 483         stonith_fence_history_cleanup(target,
 484             crm_element_value(msg, F_STONITH_CALLID) != NULL);
 485     } else if (options & st_opt_broadcast) {
 486         /* there is no clear sign atm for when a history sync
 487            is done so send a notification for anything
 488            that smells like history-sync
 489          */
 490         fenced_send_notification(T_STONITH_NOTIFY_HISTORY_SYNCED, NULL, NULL);
 491         if (crm_element_value(msg, F_STONITH_CALLID)) {
 492             /* this is coming from the stonith-API
 493             *
 494             * craft a broadcast with node's history
 495             * so that every node can merge and broadcast
 496             * what it has on top
 497             */
 498             out_history = stonith_local_history(TRUE, NULL);
 499             crm_trace("Broadcasting history to peers");
 500             stonith_send_broadcast_history(out_history,
 501                                         st_opt_broadcast | st_opt_discard_reply,
 502                                         NULL);
 503         } else if (remote_peer &&
 504                    !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
 505             xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
 506                                                 msg, LOG_NEVER);
 507 
 508             /* either a broadcast created directly upon stonith-API request
 509             * or a diff as response to such a thing
 510             *
 511             * in both cases it may have a history or not
 512             * if we have differential data
 513             * merge in what we've received and stop
 514             * otherwise broadcast what we have on top
 515             * marking as differential and merge in afterwards
 516             */
 517             if (!history || !pcmk__xe_attr_is_true(history, F_STONITH_DIFFERENTIAL)) {
 518                 GHashTable *received_history = NULL;
 519 
 520                 if (history != NULL) {
 521                     received_history = stonith_xml_history_to_list(history);
 522                 }
 523                 out_history =
 524                     stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
 525                 if (out_history) {
 526                     crm_trace("Broadcasting history-diff to peers");
 527                     pcmk__xe_set_bool_attr(out_history, F_STONITH_DIFFERENTIAL, true);
 528                     stonith_send_broadcast_history(out_history,
 529                         st_opt_broadcast | st_opt_discard_reply,
 530                         NULL);
 531                 } else {
 532                     crm_trace("History-diff is empty - skip broadcast");
 533                 }
 534             }
 535         } else {
 536             crm_trace("Skipping history-query-broadcast (%s%s)"
 537                       " we sent ourselves",
 538                       remote_peer?"remote-peer=":"local-ipc",
 539                       remote_peer?remote_peer:"");
 540         }
 541     } else {
 542         /* plain history request */
 543         crm_trace("Looking for operations on %s in %p", target,
 544                   stonith_remote_op_list);
 545         *output = stonith_local_history(FALSE, target);
 546     }
 547     free_xml(out_history);
 548 }
/* */
root/daemons/fenced/fenced_history.c

DEFINITIONS
