root/daemons/fenced/fenced_history.c

/* */

DEFINITIONS

1. stonith_send_broadcast_history
2. stonith_remove_history_entry
3. stonith_fence_history_cleanup
4. cmp_op_by_completion
5. remove_completed_remote_op
6. stonith_fence_history_trim
7. stonith_xml_history_to_list
8. stonith_local_history_diff_and_merge
9. stonith_local_history
10. stonith_fence_history

   1 /*
   2  * Copyright 2009-2025 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdio.h>
  13 #include <unistd.h>
  14 #include <stdlib.h>
  15 
  16 #include <libxml/tree.h>                // xmlNode
  17 
  18 #include <crm/crm.h>
  19 #include <crm/common/ipc.h>
  20 #include <crm/common/ipc_internal.h>
  21 #include <crm/cluster/internal.h>
  22 
  23 #include <crm/stonith-ng.h>
  24 #include <crm/fencing/internal.h>
  25 #include <crm/common/xml.h>
  26 #include <crm/common/xml_internal.h>
  27 
  28 #include <pacemaker-fenced.h>
  29 
  30 #define MAX_STONITH_HISTORY 500
  31 
  32 /*!
  33  * \internal
  34  * \brief Send a broadcast to all nodes to trigger cleanup or
  35  *        history synchronisation
  36  *
  37  * \param[in] history   Optional history to be attached
  38  * \param[in] callopts  We control cleanup via a flag in the callopts
  39  * \param[in] target    Cleanup can be limited to certain fence-targets
  40  */
  41 static void
  42 stonith_send_broadcast_history(xmlNode *history,
     /*  */
  43                                int callopts,
  44                                const char *target)
  45 {
  46     xmlNode *bcast = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND);
  47     xmlNode *wrapper = pcmk__xe_create(bcast, PCMK__XE_ST_CALLDATA);
  48     xmlNode *call_data = pcmk__xe_create(wrapper, __func__);
  49 
  50     crm_xml_add(bcast, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
  51     crm_xml_add(bcast, PCMK__XA_SUBT, PCMK__VALUE_BROADCAST);
  52     crm_xml_add(bcast, PCMK__XA_ST_OP, STONITH_OP_FENCE_HISTORY);
  53     crm_xml_add_int(bcast, PCMK__XA_ST_CALLOPT, callopts);
  54 
  55     pcmk__xml_copy(call_data, history);
  56     if (target != NULL) {
  57         crm_xml_add(call_data, PCMK__XA_ST_TARGET, target);
  58     }
  59 
  60     pcmk__cluster_send_message(NULL, pcmk_ipc_fenced, bcast);
  61 
  62     pcmk__xml_free(bcast);
  63 }
  64 
  65 static gboolean
  66 stonith_remove_history_entry (gpointer key,
     /*  */
  67                               gpointer value,
  68                               gpointer user_data)
  69 {
  70     remote_fencing_op_t *op = value;
  71     const char *target = (const char *) user_data;
  72 
  73     if ((op->state == st_failed) || (op->state == st_done)) {
  74         if ((target) && (strcmp(op->target, target) != 0)) {
  75             return FALSE;
  76         }
  77         return TRUE;
  78     }
  79 
  80     return FALSE; /* don't clean pending operations */
  81 }
  82 
  83 /*!
  84  * \internal
  85  * \brief Send out a cleanup broadcast or do a local history-cleanup
  86  *
  87  * \param[in] target    Cleanup can be limited to certain fence-targets
  88  * \param[in] broadcast Send out a cleanup broadcast
  89  */
  90 static void
  91 stonith_fence_history_cleanup(const char *target,
     /*  */
  92                               gboolean broadcast)
  93 {
  94     if (broadcast) {
  95         stonith_send_broadcast_history(NULL,
  96                                        st_opt_cleanup | st_opt_discard_reply,
  97                                        target);
  98         /* we'll do the local clean when we receive back our own broadcast */
  99     } else if (stonith_remote_op_list) {
 100         g_hash_table_foreach_remove(stonith_remote_op_list,
 101                              stonith_remove_history_entry,
 102                              (gpointer) target);
 103         fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY, NULL, NULL);
 104     }
 105 }
 106 
 107 /* keeping the length of fence-history within bounds
 108  * =================================================
 109  *
 110  * If things are really running wild a lot of fencing-attempts
 111  * might fill up the hash-map, eventually using up a lot
 112  * of memory and creating huge history-sync messages.
 113  * Before the history being synced across nodes at least
 114  * the reboot of a cluster-node helped keeping the
 115  * history within bounds even though not in a reliable
 116  * manner.
 117  *
 118  * stonith_remote_op_list isn't sorted for time-stamps
 119  * thus it would be kind of expensive to delete e.g.
 120  * the oldest entry if it would grow past MAX_STONITH_HISTORY
 121  * entries.
 122  * It is more efficient to purge MAX_STONITH_HISTORY/2
 123  * entries whenever the list grows beyond MAX_STONITH_HISTORY.
 124  * (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
 125  * That done on a per-node-base might raise the
 126  * probability of large syncs to occur.
 127  * Things like introducing a broadcast to purge
 128  * MAX_STONITH_HISTORY/2 entries or not sync above a certain
 129  * threshold coming to mind ...
 130  * Simplest thing though is to purge the full history
 131  * throughout the cluster once MAX_STONITH_HISTORY is reached.
 132  * On the other hand this leads to purging the history in
 133  * situations where it would be handy to have it probably.
 134  */
 135 
 136 /*!
 137  * \internal
 138  * \brief Compare two remote fencing operations by status and completion time
 139  *
 140  * A pending operation is ordered before a completed operation. If both
 141  * operations have completed, then the more recently completed operation is
 142  * ordered first. Two pending operations are considered equal.
 143  *
 144  * \param[in] a  First \c remote_fencing_op_t to compare
 145  * \param[in] b  Second \c remote_fencing_op_t to compare
 146  *
 147  * \return Standard comparison result (a negative integer if \p a is lesser,
 148  *         0 if the values are equal, and a positive integer if \p a is greater)
 149  */
 150 static gint
 151 cmp_op_by_completion(gconstpointer a, gconstpointer b)
     /*  */
 152 {
 153     const remote_fencing_op_t *op1 = a;
 154     const remote_fencing_op_t *op2 = b;
 155     bool op1_pending = stonith__op_state_pending(op1->state);
 156     bool op2_pending = stonith__op_state_pending(op2->state);
 157 
 158     if (op1_pending && op2_pending) {
 159         return 0;
 160     }
 161     if (op1_pending) {
 162         return -1;
 163     }
 164     if (op2_pending) {
 165         return 1;
 166     }
 167     if (op1->completed > op2->completed) {
 168         return -1;
 169     }
 170     if (op1->completed < op2->completed) {
 171         return 1;
 172     }
 173     if (op1->completed_nsec > op2->completed_nsec) {
 174         return -1;
 175     }
 176     if (op1->completed_nsec < op2->completed_nsec) {
 177         return 1;
 178     }
 179     return 0;
 180 }
 181 
 182 /*!
 183  * \internal
 184  * \brief Remove a completed operation from \c stonith_remote_op_list
 185  *
 186  * \param[in] data       \c remote_fencing_op_t to remove
 187  * \param[in] user_data  Ignored
 188  */
 189 static void
 190 remove_completed_remote_op(gpointer data, gpointer user_data)
     /*  */
 191 {
 192     const remote_fencing_op_t *op = data;
 193 
 194     if (!stonith__op_state_pending(op->state)) {
 195         g_hash_table_remove(stonith_remote_op_list, op->id);
 196     }
 197 }
 198 
 199 /*!
 200  * \internal
 201  * \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
 202  *        once over MAX_STONITH_HISTORY
 203  */
 204 void
 205 stonith_fence_history_trim(void)
     /*  */
 206 {
 207     if (stonith_remote_op_list == NULL) {
 208         return;
 209     }
 210 
 211     if (g_hash_table_size(stonith_remote_op_list) > MAX_STONITH_HISTORY) {
 212         GList *ops = g_hash_table_get_values(stonith_remote_op_list);
 213 
 214         crm_trace("More than %d entries in fencing history, purging oldest "
 215                   "completed operations", MAX_STONITH_HISTORY);
 216 
 217         ops = g_list_sort(ops, cmp_op_by_completion);
 218 
 219         // Always keep pending ops regardless of number of entries
 220         g_list_foreach(g_list_nth(ops, MAX_STONITH_HISTORY / 2),
 221                        remove_completed_remote_op, NULL);
 222 
 223         // No need for a notification after purging old data
 224         g_list_free(ops);
 225     }
 226 }
 227 
 228 /*!
 229  * \internal
 230  * \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
 231  *
 232  * \param[in] history   Fence-history in xml
 233  *
 234  * \return Fence-history as hash-table
 235  */
 236 static GHashTable *
 237 stonith_xml_history_to_list(const xmlNode *history)
     /*  */
 238 {
 239     xmlNode *xml_op = NULL;
 240     GHashTable *rv = NULL;
 241 
 242     init_stonith_remote_op_hash_table(&rv);
 243 
 244     CRM_LOG_ASSERT(rv != NULL);
 245 
 246     for (xml_op = pcmk__xe_first_child(history, NULL, NULL, NULL);
 247          xml_op != NULL; xml_op = pcmk__xe_next(xml_op, NULL)) {
 248 
 249         remote_fencing_op_t *op = NULL;
 250         char *id = crm_element_value_copy(xml_op, PCMK__XA_ST_REMOTE_OP);
 251         int state;
 252         int exit_status = CRM_EX_OK;
 253         int execution_status = PCMK_EXEC_DONE;
 254         long long completed;
 255         long long completed_nsec = 0L;
 256 
 257         if (!id) {
 258             crm_warn("Malformed fencing history received from peer");
 259             continue;
 260         }
 261 
 262         crm_trace("Attaching op %s to hashtable", id);
 263 
 264         op = pcmk__assert_alloc(1, sizeof(remote_fencing_op_t));
 265 
 266         op->id = id;
 267         op->target = crm_element_value_copy(xml_op, PCMK__XA_ST_TARGET);
 268         op->action = crm_element_value_copy(xml_op, PCMK__XA_ST_DEVICE_ACTION);
 269         op->originator = crm_element_value_copy(xml_op, PCMK__XA_ST_ORIGIN);
 270         op->delegate = crm_element_value_copy(xml_op, PCMK__XA_ST_DELEGATE);
 271         op->client_name = crm_element_value_copy(xml_op,
 272                                                  PCMK__XA_ST_CLIENTNAME);
 273         crm_element_value_ll(xml_op, PCMK__XA_ST_DATE, &completed);
 274         op->completed = (time_t) completed;
 275         crm_element_value_ll(xml_op, PCMK__XA_ST_DATE_NSEC, &completed_nsec);
 276         op->completed_nsec = completed_nsec;
 277         crm_element_value_int(xml_op, PCMK__XA_ST_STATE, &state);
 278         op->state = (enum op_state) state;
 279 
 280         /* @COMPAT We can't use stonith__xe_get_result() here because
 281          * fencers <2.1.3 didn't include results, leading it to assume an error
 282          * status. Instead, set an unknown status in that case.
 283          */
 284         if ((crm_element_value_int(xml_op, PCMK__XA_RC_CODE, &exit_status) < 0)
 285             || (crm_element_value_int(xml_op, PCMK__XA_OP_STATUS,
 286                                       &execution_status) < 0)) {
 287             exit_status = CRM_EX_INDETERMINATE;
 288             execution_status = PCMK_EXEC_UNKNOWN;
 289         }
 290         pcmk__set_result(&op->result, exit_status, execution_status,
 291                          crm_element_value(xml_op, PCMK_XA_EXIT_REASON));
 292         pcmk__set_result_output(&op->result,
 293                                 crm_element_value_copy(xml_op,
 294                                                        PCMK__XA_ST_OUTPUT),
 295                                 NULL);
 296 
 297 
 298         g_hash_table_replace(rv, id, op);
 299         CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
 300     }
 301 
 302     return rv;
 303 }
 304 
 305 /*!
 306  * \internal
 307  * \brief Craft xml difference between local fence-history and a history
 308  *        coming from remote, and merge the remote history into the local
 309  *
 310  * \param[in,out] remote_history  Fence-history as hash-table (may be NULL)
 311  * \param[in]     add_id          If crafting the answer for an API
 312  *                                history-request there is no need for the id
 313  * \param[in]     target          Optionally limit to certain fence-target
 314  *
 315  * \return The fence-history as xml
 316  */
 317 static xmlNode *
 318 stonith_local_history_diff_and_merge(GHashTable *remote_history,
     /*  */
 319                                      gboolean add_id, const char *target)
 320 {
 321     xmlNode *history = NULL;
 322     GHashTableIter iter;
 323     remote_fencing_op_t *op = NULL;
 324     gboolean updated = FALSE;
 325     int cnt = 0;
 326 
 327     if (stonith_remote_op_list) {
 328             char *id = NULL;
 329 
 330             history = pcmk__xe_create(NULL, PCMK__XE_ST_HISTORY);
 331 
 332             g_hash_table_iter_init(&iter, stonith_remote_op_list);
 333             while (g_hash_table_iter_next(&iter, (void **)&id, (void **)&op)) {
 334                 xmlNode *entry = NULL;
 335 
 336                 if (remote_history) {
 337                     remote_fencing_op_t *remote_op =
 338                         g_hash_table_lookup(remote_history, op->id);
 339 
 340                     if (remote_op) {
 341                         if (stonith__op_state_pending(op->state)
 342                             && !stonith__op_state_pending(remote_op->state)) {
 343 
 344                             crm_debug("Updating outdated pending operation %.8s "
 345                                       "(state=%s) according to the one (state=%s) from "
 346                                       "remote peer history",
 347                                       op->id, stonith__op_state_text(op->state),
 348                                       stonith__op_state_text(remote_op->state));
 349 
 350                             g_hash_table_steal(remote_history, op->id);
 351                             op->id = remote_op->id;
 352                             remote_op->id = id;
 353                             g_hash_table_iter_replace(&iter, remote_op);
 354 
 355                             updated = TRUE;
 356                             continue; /* skip outdated entries */
 357 
 358                         } else if (!stonith__op_state_pending(op->state)
 359                                    && stonith__op_state_pending(remote_op->state)) {
 360 
 361                             crm_debug("Broadcasting operation %.8s (state=%s) to "
 362                                       "update the outdated pending one "
 363                                       "(state=%s) in remote peer history",
 364                                       op->id, stonith__op_state_text(op->state),
 365                                       stonith__op_state_text(remote_op->state));
 366 
 367                             g_hash_table_remove(remote_history, op->id);
 368 
 369                         } else {
 370                             g_hash_table_remove(remote_history, op->id);
 371                             continue; /* skip entries broadcasted already */
 372                         }
 373                     }
 374                 }
 375 
 376                 if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
 377                     continue;
 378                 }
 379 
 380                 cnt++;
 381                 crm_trace("Attaching op %s", op->id);
 382                 entry = pcmk__xe_create(history, STONITH_OP_EXEC);
 383                 if (add_id) {
 384                     crm_xml_add(entry, PCMK__XA_ST_REMOTE_OP, op->id);
 385                 }
 386                 crm_xml_add(entry, PCMK__XA_ST_TARGET, op->target);
 387                 crm_xml_add(entry, PCMK__XA_ST_DEVICE_ACTION, op->action);
 388                 crm_xml_add(entry, PCMK__XA_ST_ORIGIN, op->originator);
 389                 crm_xml_add(entry, PCMK__XA_ST_DELEGATE, op->delegate);
 390                 crm_xml_add(entry, PCMK__XA_ST_CLIENTNAME, op->client_name);
 391                 crm_xml_add_ll(entry, PCMK__XA_ST_DATE, op->completed);
 392                 crm_xml_add_ll(entry, PCMK__XA_ST_DATE_NSEC,
 393                                op->completed_nsec);
 394                 crm_xml_add_int(entry, PCMK__XA_ST_STATE, op->state);
 395                 stonith__xe_set_result(entry, &op->result);
 396             }
 397     }
 398 
 399     if (remote_history) {
 400         init_stonith_remote_op_hash_table(&stonith_remote_op_list);
 401 
 402         updated |= g_hash_table_size(remote_history);
 403 
 404         g_hash_table_iter_init(&iter, remote_history);
 405         while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
 406             if (stonith__op_state_pending(op->state) &&
 407                 pcmk__str_eq(op->originator, fenced_get_local_node(),
 408                              pcmk__str_casei)) {
 409 
 410                 crm_warn("Failing pending operation %.8s originated by us but "
 411                          "known only from peer history", op->id);
 412                 op->state = st_failed;
 413                 set_fencing_completed(op);
 414 
 415                 /* CRM_EX_EXPIRED + PCMK_EXEC_INVALID prevents finalize_op()
 416                  * from setting a delegate
 417                  */
 418                 pcmk__set_result(&op->result, CRM_EX_EXPIRED, PCMK_EXEC_INVALID,
 419                                  "Initiated by earlier fencer "
 420                                  "process and presumed failed");
 421                 fenced_broadcast_op_result(op, false);
 422             }
 423 
 424             g_hash_table_iter_steal(&iter);
 425             g_hash_table_replace(stonith_remote_op_list, op->id, op);
 426             /* we could trim the history here but if we bail
 427              * out after trim we might miss more recent entries
 428              * of those that might still be in the list
 429              * if we don't bail out trimming once is more
 430              * efficient and memory overhead is minimal as
 431              * we are just moving pointers from one hash to
 432              * another
 433              */
 434         }
 435 
 436         g_hash_table_destroy(remote_history); /* remove what is left */
 437     }
 438 
 439     if (updated) {
 440         stonith_fence_history_trim();
 441         fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY, NULL, NULL);
 442     }
 443 
 444     if (cnt == 0) {
 445         pcmk__xml_free(history);
 446         return NULL;
 447     } else {
 448         return history;
 449     }
 450 }
 451 
 452 /*!
 453  * \internal
 454  * \brief Craft xml from the local fence-history
 455  *
 456  * \param[in] add_id            If crafting the answer for an API
 457  *                              history-request there is no need for the id
 458  * \param[in] target            Optionally limit to certain fence-target
 459  *
 460  * \return The fence-history as xml
 461  */
 462 static xmlNode *
 463 stonith_local_history(gboolean add_id, const char *target)
     /*  */
 464 {
 465     return stonith_local_history_diff_and_merge(NULL, add_id, target);
 466 }
 467 
 468 /*!
 469  * \internal
 470  * \brief Handle fence-history messages (from API or coming in as broadcasts)
 471  *
 472  * \param[in,out] msg          Request XML
 473  * \param[out]    output       Where to set local history, if requested
 474  * \param[in]     remote_peer  If broadcast, peer that sent it
 475  * \param[in]     options      Call options from the request
 476  */
 477 void
 478 stonith_fence_history(xmlNode *msg, xmlNode **output,
     /*  */
 479                       const char *remote_peer, int options)
 480 {
 481     const char *target = NULL;
 482     xmlNode *dev = pcmk__xpath_find_one(msg->doc,
 483                                         "//*[@" PCMK__XA_ST_TARGET "]",
 484                                         LOG_NEVER);
 485     xmlNode *out_history = NULL;
 486 
 487     if (dev) {
 488         target = crm_element_value(dev, PCMK__XA_ST_TARGET);
 489         if (target && (options & st_opt_cs_nodeid)) {
 490             int nodeid;
 491             pcmk__node_status_t *node = NULL;
 492 
 493             pcmk__scan_min_int(target, &nodeid, 0);
 494             node = pcmk__search_node_caches(nodeid, NULL, NULL,
 495                                             pcmk__node_search_any
 496                                             |pcmk__node_search_cluster_cib);
 497             if (node != NULL) {
 498                 target = node->name;
 499             }
 500         }
 501     }
 502 
 503     if (options & st_opt_cleanup) {
 504         const char *call_id = crm_element_value(msg, PCMK__XA_ST_CALLID);
 505 
 506         crm_trace("Cleaning up operations on %s in %p", target,
 507                   stonith_remote_op_list);
 508         stonith_fence_history_cleanup(target, (call_id != NULL));
 509 
 510     } else if (options & st_opt_broadcast) {
 511         /* there is no clear sign atm for when a history sync
 512            is done so send a notification for anything
 513            that smells like history-sync
 514          */
 515         fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED, NULL,
 516                                  NULL);
 517         if (crm_element_value(msg, PCMK__XA_ST_CALLID) != NULL) {
 518             /* this is coming from the stonith-API
 519             *
 520             * craft a broadcast with node's history
 521             * so that every node can merge and broadcast
 522             * what it has on top
 523             */
 524             out_history = stonith_local_history(TRUE, NULL);
 525             crm_trace("Broadcasting history to peers");
 526             stonith_send_broadcast_history(out_history,
 527                                         st_opt_broadcast | st_opt_discard_reply,
 528                                         NULL);
 529         } else if (remote_peer &&
 530                    !pcmk__str_eq(remote_peer, fenced_get_local_node(),
 531                                  pcmk__str_casei)) {
 532             xmlNode *history = pcmk__xpath_find_one(msg->doc,
 533                                                     "//" PCMK__XE_ST_HISTORY,
 534                                                     LOG_NEVER);
 535 
 536             /* either a broadcast created directly upon stonith-API request
 537             * or a diff as response to such a thing
 538             *
 539             * in both cases it may have a history or not
 540             * if we have differential data
 541             * merge in what we've received and stop
 542             * otherwise broadcast what we have on top
 543             * marking as differential and merge in afterwards
 544             */
 545             if (!history
 546                 || !pcmk__xe_attr_is_true(history, PCMK__XA_ST_DIFFERENTIAL)) {
 547 
 548                 GHashTable *received_history = NULL;
 549 
 550                 if (history != NULL) {
 551                     received_history = stonith_xml_history_to_list(history);
 552                 }
 553                 out_history =
 554                     stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
 555                 if (out_history) {
 556                     crm_trace("Broadcasting history-diff to peers");
 557                     pcmk__xe_set_bool_attr(out_history,
 558                                            PCMK__XA_ST_DIFFERENTIAL, true);
 559                     stonith_send_broadcast_history(out_history,
 560                         st_opt_broadcast | st_opt_discard_reply,
 561                         NULL);
 562                 } else {
 563                     crm_trace("History-diff is empty - skip broadcast");
 564                 }
 565             }
 566         } else {
 567             crm_trace("Skipping history-query-broadcast (%s%s)"
 568                       " we sent ourselves",
 569                       remote_peer?"remote-peer=":"local-ipc",
 570                       remote_peer?remote_peer:"");
 571         }
 572     } else {
 573         /* plain history request */
 574         crm_trace("Looking for operations on %s in %p", target,
 575                   stonith_remote_op_list);
 576         *output = stonith_local_history(FALSE, target);
 577     }
 578     pcmk__xml_free(out_history);
 579 }

/* */