daemons/fenced/fenced

/* */
This source file includes following definitions.
stonith_send_broadcast_history
stonith_remove_history_entry
stonith_fence_history_cleanup
cmp_op_by_completion
remove_completed_remote_op
stonith_fence_history_trim
stonith_xml_history_to_list
stonith_local_history_diff_and_merge
stonith_local_history
stonith_fence_history
   1 /*
   2  * Copyright 2009-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdio.h>
  13 #include <unistd.h>
  14 #include <stdlib.h>
  15 
  16 #include <crm/crm.h>
  17 #include <crm/common/ipc.h>
  18 #include <crm/common/ipc_internal.h>
  19 #include <crm/cluster/internal.h>
  20 
  21 #include <crm/stonith-ng.h>
  22 #include <crm/fencing/internal.h>
  23 #include <crm/common/xml.h>
  24 #include <crm/common/xml_internal.h>
  25 
  26 #include <pacemaker-fenced.h>
  27 
  28 #define MAX_STONITH_HISTORY 500
  29 
  30 /*!
  31  * \internal
  32  * \brief Send a broadcast to all nodes to trigger cleanup or
  33  *        history synchronisation
  34  *
  35  * \param[in] history   Optional history to be attached
  36  * \param[in] callopts  We control cleanup via a flag in the callopts
  37  * \param[in] target    Cleanup can be limited to certain fence-targets
  38  */
  39 static void
  40 stonith_send_broadcast_history(xmlNode *history,
     /*  */
  41                                int callopts,
  42                                const char *target)
  43 {
  44     xmlNode *bcast = pcmk__xe_create(NULL, PCMK__XE_STONITH_COMMAND);
  45     xmlNode *wrapper = pcmk__xe_create(bcast, PCMK__XE_ST_CALLDATA);
  46     xmlNode *call_data = pcmk__xe_create(wrapper, __func__);
  47 
  48     crm_xml_add(bcast, PCMK__XA_T, PCMK__VALUE_STONITH_NG);
  49     crm_xml_add(bcast, PCMK__XA_SUBT, PCMK__VALUE_BROADCAST);
  50     crm_xml_add(bcast, PCMK__XA_ST_OP, STONITH_OP_FENCE_HISTORY);
  51     crm_xml_add_int(bcast, PCMK__XA_ST_CALLOPT, callopts);
  52 
  53     pcmk__xml_copy(call_data, history);
  54     if (target != NULL) {
  55         crm_xml_add(call_data, PCMK__XA_ST_TARGET, target);
  56     }
  57 
  58     pcmk__cluster_send_message(NULL, crm_msg_stonith_ng, bcast);
  59 
  60     free_xml(bcast);
  61 }
  62 
  63 static gboolean
  64 stonith_remove_history_entry (gpointer key,
     /*  */
  65                               gpointer value,
  66                               gpointer user_data)
  67 {
  68     remote_fencing_op_t *op = value;
  69     const char *target = (const char *) user_data;
  70 
  71     if ((op->state == st_failed) || (op->state == st_done)) {
  72         if ((target) && (strcmp(op->target, target) != 0)) {
  73             return FALSE;
  74         }
  75         return TRUE;
  76     }
  77 
  78     return FALSE; /* don't clean pending operations */
  79 }
  80 
  81 /*!
  82  * \internal
  83  * \brief Send out a cleanup broadcast or do a local history-cleanup
  84  *
  85  * \param[in] target    Cleanup can be limited to certain fence-targets
  86  * \param[in] broadcast Send out a cleanup broadcast
  87  */
  88 static void
  89 stonith_fence_history_cleanup(const char *target,
     /*  */
  90                               gboolean broadcast)
  91 {
  92     if (broadcast) {
  93         stonith_send_broadcast_history(NULL,
  94                                        st_opt_cleanup | st_opt_discard_reply,
  95                                        target);
  96         /* we'll do the local clean when we receive back our own broadcast */
  97     } else if (stonith_remote_op_list) {
  98         g_hash_table_foreach_remove(stonith_remote_op_list,
  99                              stonith_remove_history_entry,
 100                              (gpointer) target);
 101         fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY, NULL, NULL);
 102     }
 103 }
 104 
 105 /* keeping the length of fence-history within bounds
 106  * =================================================
 107  *
 108  * If things are really running wild a lot of fencing-attempts
 109  * might fill up the hash-map, eventually using up a lot
 110  * of memory and creating huge history-sync messages.
 111  * Before the history being synced across nodes at least
 112  * the reboot of a cluster-node helped keeping the
 113  * history within bounds even though not in a reliable
 114  * manner.
 115  *
 116  * stonith_remote_op_list isn't sorted for time-stamps
 117  * thus it would be kind of expensive to delete e.g.
 118  * the oldest entry if it would grow past MAX_STONITH_HISTORY
 119  * entries.
 120  * It is more efficient to purge MAX_STONITH_HISTORY/2
 121  * entries whenever the list grows beyond MAX_STONITH_HISTORY.
 122  * (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
 123  * That done on a per-node-base might raise the
 124  * probability of large syncs to occur.
 125  * Things like introducing a broadcast to purge
 126  * MAX_STONITH_HISTORY/2 entries or not sync above a certain
 127  * threshold coming to mind ...
 128  * Simplest thing though is to purge the full history
 129  * throughout the cluster once MAX_STONITH_HISTORY is reached.
 130  * On the other hand this leads to purging the history in
 131  * situations where it would be handy to have it probably.
 132  */
 133 
 134 /*!
 135  * \internal
 136  * \brief Compare two remote fencing operations by status and completion time
 137  *
 138  * A pending operation is ordered before a completed operation. If both
 139  * operations have completed, then the more recently completed operation is
 140  * ordered first. Two pending operations are considered equal.
 141  *
 142  * \param[in] a  First \c remote_fencing_op_t to compare
 143  * \param[in] b  Second \c remote_fencing_op_t to compare
 144  *
 145  * \return Standard comparison result (a negative integer if \p a is lesser,
 146  *         0 if the values are equal, and a positive integer if \p a is greater)
 147  */
 148 static gint
 149 cmp_op_by_completion(gconstpointer a, gconstpointer b)
     /*  */
 150 {
 151     const remote_fencing_op_t *op1 = a;
 152     const remote_fencing_op_t *op2 = b;
 153     bool op1_pending = stonith__op_state_pending(op1->state);
 154     bool op2_pending = stonith__op_state_pending(op2->state);
 155 
 156     if (op1_pending && op2_pending) {
 157         return 0;
 158     }
 159     if (op1_pending) {
 160         return -1;
 161     }
 162     if (op2_pending) {
 163         return 1;
 164     }
 165     if (op1->completed > op2->completed) {
 166         return -1;
 167     }
 168     if (op1->completed < op2->completed) {
 169         return 1;
 170     }
 171     if (op1->completed_nsec > op2->completed_nsec) {
 172         return -1;
 173     }
 174     if (op1->completed_nsec < op2->completed_nsec) {
 175         return 1;
 176     }
 177     return 0;
 178 }
 179 
 180 /*!
 181  * \internal
 182  * \brief Remove a completed operation from \c stonith_remote_op_list
 183  *
 184  * \param[in] data       \c remote_fencing_op_t to remove
 185  * \param[in] user_data  Ignored
 186  */
 187 static void
 188 remove_completed_remote_op(gpointer data, gpointer user_data)
     /*  */
 189 {
 190     const remote_fencing_op_t *op = data;
 191 
 192     if (!stonith__op_state_pending(op->state)) {
 193         g_hash_table_remove(stonith_remote_op_list, op->id);
 194     }
 195 }
 196 
 197 /*!
 198  * \internal
 199  * \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
 200  *        once over MAX_STONITH_HISTORY
 201  */
 202 void
 203 stonith_fence_history_trim(void)
     /*  */
 204 {
 205     if (stonith_remote_op_list == NULL) {
 206         return;
 207     }
 208 
 209     if (g_hash_table_size(stonith_remote_op_list) > MAX_STONITH_HISTORY) {
 210         GList *ops = g_hash_table_get_values(stonith_remote_op_list);
 211 
 212         crm_trace("More than %d entries in fencing history, purging oldest "
 213                   "completed operations", MAX_STONITH_HISTORY);
 214 
 215         ops = g_list_sort(ops, cmp_op_by_completion);
 216 
 217         // Always keep pending ops regardless of number of entries
 218         g_list_foreach(g_list_nth(ops, MAX_STONITH_HISTORY / 2),
 219                        remove_completed_remote_op, NULL);
 220 
 221         // No need for a notification after purging old data
 222         g_list_free(ops);
 223     }
 224 }
 225 
 226 /*!
 227  * \internal
 228  * \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
 229  *
 230  * \param[in] history   Fence-history in xml
 231  *
 232  * \return Fence-history as hash-table
 233  */
 234 static GHashTable *
 235 stonith_xml_history_to_list(const xmlNode *history)
     /*  */
 236 {
 237     xmlNode *xml_op = NULL;
 238     GHashTable *rv = NULL;
 239 
 240     init_stonith_remote_op_hash_table(&rv);
 241 
 242     CRM_LOG_ASSERT(rv != NULL);
 243 
 244     for (xml_op = pcmk__xe_first_child(history, NULL, NULL, NULL);
 245          xml_op != NULL; xml_op = pcmk__xe_next(xml_op)) {
 246 
 247         remote_fencing_op_t *op = NULL;
 248         char *id = crm_element_value_copy(xml_op, PCMK__XA_ST_REMOTE_OP);
 249         int state;
 250         int exit_status = CRM_EX_OK;
 251         int execution_status = PCMK_EXEC_DONE;
 252         long long completed;
 253         long long completed_nsec = 0L;
 254 
 255         if (!id) {
 256             crm_warn("Malformed fencing history received from peer");
 257             continue;
 258         }
 259 
 260         crm_trace("Attaching op %s to hashtable", id);
 261 
 262         op = pcmk__assert_alloc(1, sizeof(remote_fencing_op_t));
 263 
 264         op->id = id;
 265         op->target = crm_element_value_copy(xml_op, PCMK__XA_ST_TARGET);
 266         op->action = crm_element_value_copy(xml_op, PCMK__XA_ST_DEVICE_ACTION);
 267         op->originator = crm_element_value_copy(xml_op, PCMK__XA_ST_ORIGIN);
 268         op->delegate = crm_element_value_copy(xml_op, PCMK__XA_ST_DELEGATE);
 269         op->client_name = crm_element_value_copy(xml_op,
 270                                                  PCMK__XA_ST_CLIENTNAME);
 271         crm_element_value_ll(xml_op, PCMK__XA_ST_DATE, &completed);
 272         op->completed = (time_t) completed;
 273         crm_element_value_ll(xml_op, PCMK__XA_ST_DATE_NSEC, &completed_nsec);
 274         op->completed_nsec = completed_nsec;
 275         crm_element_value_int(xml_op, PCMK__XA_ST_STATE, &state);
 276         op->state = (enum op_state) state;
 277 
 278         /* @COMPAT We can't use stonith__xe_get_result() here because
 279          * fencers <2.1.3 didn't include results, leading it to assume an error
 280          * status. Instead, set an unknown status in that case.
 281          */
 282         if ((crm_element_value_int(xml_op, PCMK__XA_RC_CODE, &exit_status) < 0)
 283             || (crm_element_value_int(xml_op, PCMK__XA_OP_STATUS,
 284                                       &execution_status) < 0)) {
 285             exit_status = CRM_EX_INDETERMINATE;
 286             execution_status = PCMK_EXEC_UNKNOWN;
 287         }
 288         pcmk__set_result(&op->result, exit_status, execution_status,
 289                          crm_element_value(xml_op, PCMK_XA_EXIT_REASON));
 290         pcmk__set_result_output(&op->result,
 291                                 crm_element_value_copy(xml_op,
 292                                                        PCMK__XA_ST_OUTPUT),
 293                                 NULL);
 294 
 295 
 296         g_hash_table_replace(rv, id, op);
 297         CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
 298     }
 299 
 300     return rv;
 301 }
 302 
 303 /*!
 304  * \internal
 305  * \brief Craft xml difference between local fence-history and a history
 306  *        coming from remote, and merge the remote history into the local
 307  *
 308  * \param[in,out] remote_history  Fence-history as hash-table (may be NULL)
 309  * \param[in]     add_id          If crafting the answer for an API
 310  *                                history-request there is no need for the id
 311  * \param[in]     target          Optionally limit to certain fence-target
 312  *
 313  * \return The fence-history as xml
 314  */
 315 static xmlNode *
 316 stonith_local_history_diff_and_merge(GHashTable *remote_history,
     /*  */
 317                                      gboolean add_id, const char *target)
 318 {
 319     xmlNode *history = NULL;
 320     GHashTableIter iter;
 321     remote_fencing_op_t *op = NULL;
 322     gboolean updated = FALSE;
 323     int cnt = 0;
 324 
 325     if (stonith_remote_op_list) {
 326             char *id = NULL;
 327 
 328             history = pcmk__xe_create(NULL, PCMK__XE_ST_HISTORY);
 329 
 330             g_hash_table_iter_init(&iter, stonith_remote_op_list);
 331             while (g_hash_table_iter_next(&iter, (void **)&id, (void **)&op)) {
 332                 xmlNode *entry = NULL;
 333 
 334                 if (remote_history) {
 335                     remote_fencing_op_t *remote_op =
 336                         g_hash_table_lookup(remote_history, op->id);
 337 
 338                     if (remote_op) {
 339                         if (stonith__op_state_pending(op->state)
 340                             && !stonith__op_state_pending(remote_op->state)) {
 341 
 342                             crm_debug("Updating outdated pending operation %.8s "
 343                                       "(state=%s) according to the one (state=%s) from "
 344                                       "remote peer history",
 345                                       op->id, stonith_op_state_str(op->state),
 346                                       stonith_op_state_str(remote_op->state));
 347 
 348                             g_hash_table_steal(remote_history, op->id);
 349                             op->id = remote_op->id;
 350                             remote_op->id = id;
 351                             g_hash_table_iter_replace(&iter, remote_op);
 352 
 353                             updated = TRUE;
 354                             continue; /* skip outdated entries */
 355 
 356                         } else if (!stonith__op_state_pending(op->state)
 357                                    && stonith__op_state_pending(remote_op->state)) {
 358 
 359                             crm_debug("Broadcasting operation %.8s (state=%s) to "
 360                                       "update the outdated pending one "
 361                                       "(state=%s) in remote peer history",
 362                                       op->id, stonith_op_state_str(op->state),
 363                                       stonith_op_state_str(remote_op->state));
 364 
 365                             g_hash_table_remove(remote_history, op->id);
 366 
 367                         } else {
 368                             g_hash_table_remove(remote_history, op->id);
 369                             continue; /* skip entries broadcasted already */
 370                         }
 371                     }
 372                 }
 373 
 374                 if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
 375                     continue;
 376                 }
 377 
 378                 cnt++;
 379                 crm_trace("Attaching op %s", op->id);
 380                 entry = pcmk__xe_create(history, STONITH_OP_EXEC);
 381                 if (add_id) {
 382                     crm_xml_add(entry, PCMK__XA_ST_REMOTE_OP, op->id);
 383                 }
 384                 crm_xml_add(entry, PCMK__XA_ST_TARGET, op->target);
 385                 crm_xml_add(entry, PCMK__XA_ST_DEVICE_ACTION, op->action);
 386                 crm_xml_add(entry, PCMK__XA_ST_ORIGIN, op->originator);
 387                 crm_xml_add(entry, PCMK__XA_ST_DELEGATE, op->delegate);
 388                 crm_xml_add(entry, PCMK__XA_ST_CLIENTNAME, op->client_name);
 389                 crm_xml_add_ll(entry, PCMK__XA_ST_DATE, op->completed);
 390                 crm_xml_add_ll(entry, PCMK__XA_ST_DATE_NSEC,
 391                                op->completed_nsec);
 392                 crm_xml_add_int(entry, PCMK__XA_ST_STATE, op->state);
 393                 stonith__xe_set_result(entry, &op->result);
 394             }
 395     }
 396 
 397     if (remote_history) {
 398         init_stonith_remote_op_hash_table(&stonith_remote_op_list);
 399 
 400         updated |= g_hash_table_size(remote_history);
 401 
 402         g_hash_table_iter_init(&iter, remote_history);
 403         while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
 404             if (stonith__op_state_pending(op->state) &&
 405                 pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
 406 
 407                 crm_warn("Failing pending operation %.8s originated by us but "
 408                          "known only from peer history", op->id);
 409                 op->state = st_failed;
 410                 set_fencing_completed(op);
 411 
 412                 /* CRM_EX_EXPIRED + PCMK_EXEC_INVALID prevents finalize_op()
 413                  * from setting a delegate
 414                  */
 415                 pcmk__set_result(&op->result, CRM_EX_EXPIRED, PCMK_EXEC_INVALID,
 416                                  "Initiated by earlier fencer "
 417                                  "process and presumed failed");
 418                 fenced_broadcast_op_result(op, false);
 419             }
 420 
 421             g_hash_table_iter_steal(&iter);
 422             g_hash_table_replace(stonith_remote_op_list, op->id, op);
 423             /* we could trim the history here but if we bail
 424              * out after trim we might miss more recent entries
 425              * of those that might still be in the list
 426              * if we don't bail out trimming once is more
 427              * efficient and memory overhead is minimal as
 428              * we are just moving pointers from one hash to
 429              * another
 430              */
 431         }
 432 
 433         g_hash_table_destroy(remote_history); /* remove what is left */
 434     }
 435 
 436     if (updated) {
 437         stonith_fence_history_trim();
 438         fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY, NULL, NULL);
 439     }
 440 
 441     if (cnt == 0) {
 442         free_xml(history);
 443         return NULL;
 444     } else {
 445         return history;
 446     }
 447 }
 448 
 449 /*!
 450  * \internal
 451  * \brief Craft xml from the local fence-history
 452  *
 453  * \param[in] add_id            If crafting the answer for an API
 454  *                              history-request there is no need for the id
 455  * \param[in] target            Optionally limit to certain fence-target
 456  *
 457  * \return The fence-history as xml
 458  */
 459 static xmlNode *
 460 stonith_local_history(gboolean add_id, const char *target)
     /*  */
 461 {
 462     return stonith_local_history_diff_and_merge(NULL, add_id, target);
 463 }
 464 
 465 /*!
 466  * \internal
 467  * \brief Handle fence-history messages (from API or coming in as broadcasts)
 468  *
 469  * \param[in,out] msg          Request XML
 470  * \param[out]    output       Where to set local history, if requested
 471  * \param[in]     remote_peer  If broadcast, peer that sent it
 472  * \param[in]     options      Call options from the request
 473  */
 474 void
 475 stonith_fence_history(xmlNode *msg, xmlNode **output,
     /*  */
 476                       const char *remote_peer, int options)
 477 {
 478     const char *target = NULL;
 479     xmlNode *dev = get_xpath_object("//@" PCMK__XA_ST_TARGET, msg, LOG_NEVER);
 480     xmlNode *out_history = NULL;
 481 
 482     if (dev) {
 483         target = crm_element_value(dev, PCMK__XA_ST_TARGET);
 484         if (target && (options & st_opt_cs_nodeid)) {
 485             int nodeid;
 486             crm_node_t *node;
 487 
 488             pcmk__scan_min_int(target, &nodeid, 0);
 489             node = pcmk__search_node_caches(nodeid, NULL,
 490                                             pcmk__node_search_any
 491                                             |pcmk__node_search_cluster_cib);
 492             if (node) {
 493                 target = node->uname;
 494             }
 495         }
 496     }
 497 
 498     if (options & st_opt_cleanup) {
 499         const char *call_id = crm_element_value(msg, PCMK__XA_ST_CALLID);
 500 
 501         crm_trace("Cleaning up operations on %s in %p", target,
 502                   stonith_remote_op_list);
 503         stonith_fence_history_cleanup(target, (call_id != NULL));
 504 
 505     } else if (options & st_opt_broadcast) {
 506         /* there is no clear sign atm for when a history sync
 507            is done so send a notification for anything
 508            that smells like history-sync
 509          */
 510         fenced_send_notification(PCMK__VALUE_ST_NOTIFY_HISTORY_SYNCED, NULL,
 511                                  NULL);
 512         if (crm_element_value(msg, PCMK__XA_ST_CALLID) != NULL) {
 513             /* this is coming from the stonith-API
 514             *
 515             * craft a broadcast with node's history
 516             * so that every node can merge and broadcast
 517             * what it has on top
 518             */
 519             out_history = stonith_local_history(TRUE, NULL);
 520             crm_trace("Broadcasting history to peers");
 521             stonith_send_broadcast_history(out_history,
 522                                         st_opt_broadcast | st_opt_discard_reply,
 523                                         NULL);
 524         } else if (remote_peer &&
 525                    !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
 526             xmlNode *history = get_xpath_object("//" PCMK__XE_ST_HISTORY, msg,
 527                                                 LOG_NEVER);
 528 
 529             /* either a broadcast created directly upon stonith-API request
 530             * or a diff as response to such a thing
 531             *
 532             * in both cases it may have a history or not
 533             * if we have differential data
 534             * merge in what we've received and stop
 535             * otherwise broadcast what we have on top
 536             * marking as differential and merge in afterwards
 537             */
 538             if (!history
 539                 || !pcmk__xe_attr_is_true(history, PCMK__XA_ST_DIFFERENTIAL)) {
 540 
 541                 GHashTable *received_history = NULL;
 542 
 543                 if (history != NULL) {
 544                     received_history = stonith_xml_history_to_list(history);
 545                 }
 546                 out_history =
 547                     stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
 548                 if (out_history) {
 549                     crm_trace("Broadcasting history-diff to peers");
 550                     pcmk__xe_set_bool_attr(out_history,
 551                                            PCMK__XA_ST_DIFFERENTIAL, true);
 552                     stonith_send_broadcast_history(out_history,
 553                         st_opt_broadcast | st_opt_discard_reply,
 554                         NULL);
 555                 } else {
 556                     crm_trace("History-diff is empty - skip broadcast");
 557                 }
 558             }
 559         } else {
 560             crm_trace("Skipping history-query-broadcast (%s%s)"
 561                       " we sent ourselves",
 562                       remote_peer?"remote-peer=":"local-ipc",
 563                       remote_peer?remote_peer:"");
 564         }
 565     } else {
 566         /* plain history request */
 567         crm_trace("Looking for operations on %s in %p", target,
 568                   stonith_remote_op_list);
 569         *output = stonith_local_history(FALSE, target);
 570     }
 571     free_xml(out_history);
 572 }
/* */
root/daemons/fenced/fenced_history.c

DEFINITIONS
