root/daemons/fenced/fenced_history.c

/* */

DEFINITIONS

1. stonith_send_broadcast_history
2. stonith_remove_history_entry
3. stonith_fence_history_cleanup
4. op_time_sort
5. stonith_fence_history_trim
6. stonith_xml_history_to_list
7. stonith_local_history_diff
8. stonith_merge_in_history_list
9. stonith_fence_history

   1 /*
   2  * Copyright 2009-2020 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdio.h>
  13 #include <unistd.h>
  14 #include <stdlib.h>
  15 
  16 #include <crm/crm.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/common/ipc.h>
  19 #include <crm/common/ipc_internal.h>
  20 #include <crm/cluster/internal.h>
  21 
  22 #include <crm/stonith-ng.h>
  23 #include <crm/fencing/internal.h>
  24 #include <crm/common/xml.h>
  25 #include <crm/common/xml_internal.h>
  26 
  27 #include <pacemaker-fenced.h>
  28 
  29 #define MAX_STONITH_HISTORY 500
  30 
  31 /*!
  32  * \internal
  33  * \brief Send a broadcast to all nodes to trigger cleanup or
  34  *        history synchronisation
  35  *
  36  * \param[in] history   Optional history to be attached
  37  * \param[in] callopts  We control cleanup via a flag in the callopts
  38  * \param[in] target    Cleanup can be limited to certain fence-targets
  39  */
  40 static void
  41 stonith_send_broadcast_history(xmlNode *history,
     /*  */
  42                                int callopts,
  43                                const char *target)
  44 {
  45     xmlNode *bcast = create_xml_node(NULL, "stonith_command");
  46     xmlNode *data = create_xml_node(NULL, __func__);
  47 
  48     if (target) {
  49         crm_xml_add(data, F_STONITH_TARGET, target);
  50     }
  51     crm_xml_add(bcast, F_TYPE, T_STONITH_NG);
  52     crm_xml_add(bcast, F_SUBTYPE, "broadcast");
  53     crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY);
  54     crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts);
  55     if (history) {
  56         add_node_copy(data, history);
  57     }
  58     add_message_xml(bcast, F_STONITH_CALLDATA, data);
  59     send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
  60 
  61     free_xml(data);
  62     free_xml(bcast);
  63 }
  64 
  65 static gboolean
  66 stonith_remove_history_entry (gpointer key,
     /*  */
  67                               gpointer value,
  68                               gpointer user_data)
  69 {
  70     remote_fencing_op_t *op = value;
  71     const char *target = (const char *) user_data;
  72 
  73     if ((op->state == st_failed) || (op->state == st_done)) {
  74         if ((target) && (strcmp(op->target, target) != 0)) {
  75             return FALSE;
  76         }
  77         return TRUE;
  78     }
  79 
  80     return FALSE; /* don't clean pending operations */
  81 }
  82 
  83 /*!
  84  * \internal
  85  * \brief Send out a cleanup broadcast or do a local history-cleanup
  86  *
  87  * \param[in] target    Cleanup can be limited to certain fence-targets
  88  * \param[in] broadcast Send out a cleanup broadcast
  89  */
  90 static void
  91 stonith_fence_history_cleanup(const char *target,
     /*  */
  92                               gboolean broadcast)
  93 {
  94     if (broadcast) {
  95         stonith_send_broadcast_history(NULL,
  96                                        st_opt_cleanup | st_opt_discard_reply,
  97                                        target);
  98         /* we'll do the local clean when we receive back our own broadcast */
  99     } else if (stonith_remote_op_list) {
 100         g_hash_table_foreach_remove(stonith_remote_op_list,
 101                              stonith_remove_history_entry,
 102                              (gpointer) target);
 103         do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);
 104     }
 105 }
 106 
 107 /* keeping the length of fence-history within bounds
 108  * =================================================
 109  *
 110  * If things are really running wild a lot of fencing-attempts
 111  * might fill up the hash-map, eventually using up a lot
 112  * of memory and creating huge history-sync messages.
 113  * Before the history being synced across nodes at least
 114  * the reboot of a cluster-node helped keeping the
 115  * history within bounds even though not in a reliable
 116  * manner.
 117  *
 118  * stonith_remote_op_list isn't sorted for time-stamps
 119  * thus it would be kind of expensive to delete e.g.
 120  * the oldest entry if it would grow past MAX_STONITH_HISTORY
 121  * entries.
 122  * It is more efficient to purge MAX_STONITH_HISTORY/2
 123  * entries whenever the list grows beyond MAX_STONITH_HISTORY.
 124  * (sort for age + purge the MAX_STONITH_HISTORY/2 oldest)
 125  * That done on a per-node-base might raise the
 126  * probability of large syncs to occur.
 127  * Things like introducing a broadcast to purge
 128  * MAX_STONITH_HISTORY/2 entries or not sync above a certain
 129  * threshold coming to mind ...
 130  * Simplest thing though is to purge the full history
 131  * throughout the cluster once MAX_STONITH_HISTORY is reached.
 132  * On the other hand this leads to purging the history in
 133  * situations where it would be handy to have it probably.
 134  */
 135 
 136 
 137 static int
 138 op_time_sort(const void *a_voidp, const void *b_voidp)
     /*  */
 139 {
 140     const remote_fencing_op_t **a = (const remote_fencing_op_t **) a_voidp;
 141     const remote_fencing_op_t **b = (const remote_fencing_op_t **) b_voidp;
 142     gboolean a_pending = ((*a)->state != st_failed) && ((*a)->state != st_done);
 143     gboolean b_pending = ((*b)->state != st_failed) && ((*b)->state != st_done);
 144 
 145     if (a_pending && b_pending) {
 146         return 0;
 147     } else if (a_pending) {
 148         return -1;
 149     } else if (b_pending) {
 150         return 1;
 151     } else if ((*b)->completed == (*a)->completed) {
 152         return 0;
 153     } else if ((*b)->completed > (*a)->completed) {
 154         return 1;
 155     }
 156 
 157     return -1;
 158 }
 159 
 160 
 161 /*!
 162  * \internal
 163  * \brief Do a local history-trim to MAX_STONITH_HISTORY / 2 entries
 164  *        once over MAX_STONITH_HISTORY
 165  */
 166 void
 167 stonith_fence_history_trim(void)
     /*  */
 168 {
 169     guint num_ops;
 170 
 171     if (!stonith_remote_op_list) {
 172         return;
 173     }
 174     num_ops = g_hash_table_size(stonith_remote_op_list);
 175     if (num_ops > MAX_STONITH_HISTORY) {
 176         remote_fencing_op_t *ops[num_ops];
 177         remote_fencing_op_t *op = NULL;
 178         GHashTableIter iter;
 179         int i;
 180 
 181         crm_trace("Fencing History growing beyond limit of %d so purge "
 182                   "half of failed/successful attempts", MAX_STONITH_HISTORY);
 183 
 184         /* write all ops into an array */
 185         i = 0;
 186         g_hash_table_iter_init(&iter, stonith_remote_op_list);
 187         while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
 188             ops[i++] = op;
 189         }
 190         /* run quicksort over the array so that we get pending ops
 191          * first and then sorted most recent to oldest
 192          */
 193         qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort);
 194         /* purgest oldest half of the history entries */
 195         for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) {
 196             /* keep pending ops even if they shouldn't fill more than
 197              * half of our buffer
 198              */
 199             if ((ops[i]->state == st_failed) || (ops[i]->state == st_done)) {
 200                 g_hash_table_remove(stonith_remote_op_list, ops[i]->id);
 201             }
 202         }
 203         /* we've just purged valid data from the list so there is no need
 204          * to create a notification - if displayed it can stay
 205          */
 206     }
 207 }
 208 
 209 /*!
 210  * \internal
 211  * \brief Convert xml fence-history to a hash-table like stonith_remote_op_list
 212  *
 213  * \param[in] history   Fence-history in xml
 214  *
 215  * \return Fence-history as hash-table
 216  */
 217 static GHashTable *
 218 stonith_xml_history_to_list(xmlNode *history)
     /*  */
 219 {
 220     xmlNode *xml_op = NULL;
 221     GHashTable *rv = NULL;
 222 
 223     init_stonith_remote_op_hash_table(&rv);
 224 
 225     CRM_LOG_ASSERT(rv != NULL);
 226 
 227     for (xml_op = pcmk__xml_first_child(history); xml_op != NULL;
 228          xml_op = pcmk__xml_next(xml_op)) {
 229         remote_fencing_op_t *op = NULL;
 230         char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID);
 231         int state;
 232         long long completed;
 233 
 234         if (!id) {
 235             crm_warn("History to convert to hashtable has no id in entry");
 236             continue;
 237         }
 238 
 239         crm_trace("Attaching op %s to hashtable", id);
 240 
 241         op = calloc(1, sizeof(remote_fencing_op_t));
 242 
 243         op->id = id;
 244         op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET);
 245         op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION);
 246         op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN);
 247         op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE);
 248         op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME);
 249         crm_element_value_ll(xml_op, F_STONITH_DATE, &completed);
 250         op->completed = (time_t) completed;
 251         crm_element_value_int(xml_op, F_STONITH_STATE, &state);
 252         op->state = (enum op_state) state;
 253 
 254         g_hash_table_replace(rv, id, op);
 255         CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
 256     }
 257 
 258     return rv;
 259 }
 260 
 261 /*!
 262  * \internal
 263  * \brief Craft xml difference between local fence-history and a history
 264  *        coming from remote
 265  *
 266  * \param[in] remote_history    Fence-history as hash-table (may be NULL)
 267  * \param[in] add_id            If crafting the answer for an API
 268  *                              history-request there is no need for the id
 269  * \param[in] target            Optionally limit to certain fence-target
 270  *
 271  * \return The fence-history as xml
 272  */
 273 static xmlNode *
 274 stonith_local_history_diff(GHashTable *remote_history,
     /*  */
 275                            gboolean add_id,
 276                            const char *target)
 277 {
 278     xmlNode *history = NULL;
 279     int cnt = 0;
 280 
 281     if (stonith_remote_op_list) {
 282             GHashTableIter iter;
 283             remote_fencing_op_t *op = NULL;
 284 
 285             history = create_xml_node(NULL, F_STONITH_HISTORY_LIST);
 286 
 287             g_hash_table_iter_init(&iter, stonith_remote_op_list);
 288             while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
 289                 xmlNode *entry = NULL;
 290 
 291                 if (remote_history &&
 292                     g_hash_table_lookup(remote_history, op->id)) {
 293                     continue; /* skip entries broadcasted already */
 294                 }
 295 
 296                 if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
 297                     continue;
 298                 }
 299 
 300                 cnt++;
 301                 crm_trace("Attaching op %s", op->id);
 302                 entry = create_xml_node(history, STONITH_OP_EXEC);
 303                 if (add_id) {
 304                     crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id);
 305                 }
 306                 crm_xml_add(entry, F_STONITH_TARGET, op->target);
 307                 crm_xml_add(entry, F_STONITH_ACTION, op->action);
 308                 crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
 309                 crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
 310                 crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
 311                 crm_xml_add_ll(entry, F_STONITH_DATE, op->completed);
 312                 crm_xml_add_int(entry, F_STONITH_STATE, op->state);
 313             }
 314     }
 315 
 316     if (cnt == 0) {
 317         free_xml(history);
 318         return NULL;
 319     } else {
 320         return history;
 321     }
 322 }
 323 
 324 /*!
 325  * \internal
 326  * \brief Merge fence-history coming from remote into local history
 327  *
 328  * \param[in] history   Hash-table holding remote history to be merged in
 329  */
 330 static void
 331 stonith_merge_in_history_list(GHashTable *history)
     /*  */
 332 {
 333     GHashTableIter iter;
 334     remote_fencing_op_t *op = NULL;
 335     gboolean updated = FALSE;
 336 
 337     if (!history) {
 338         return;
 339     }
 340 
 341     init_stonith_remote_op_hash_table(&stonith_remote_op_list);
 342 
 343     g_hash_table_iter_init(&iter, history);
 344     while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
 345         remote_fencing_op_t *stored_op =
 346             g_hash_table_lookup(stonith_remote_op_list, op->id);
 347 
 348         if (stored_op) {
 349             continue; // Skip existent (@TODO state-merging might be desirable)
 350         }
 351 
 352         updated = TRUE;
 353         g_hash_table_iter_steal(&iter);
 354 
 355         if ((op->state != st_failed) &&
 356             (op->state != st_done) &&
 357             pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
 358             crm_warn("received pending action we are supposed to be the "
 359                      "owner but it's not in our records -> fail it");
 360             op->state = st_failed;
 361             op->completed = time(NULL);
 362             /* use -EHOSTUNREACH to not introduce a new return-code that might
 363                trigger unexpected results at other places and to prevent
 364                remote_op_done from setting the delegate if not present
 365              */
 366             stonith_bcast_result_to_peers(op, -EHOSTUNREACH, FALSE);
 367         }
 368 
 369         g_hash_table_insert(stonith_remote_op_list, op->id, op);
 370         /* we could trim the history here but if we bail
 371          * out after trim we might miss more recent entries
 372          * of those that might still be in the list
 373          * if we don't bail out trimming once is more
 374          * efficient and memory overhead is minimal as
 375          * we are just moving pointers from one hash to
 376          * another
 377          */
 378     }
 379     stonith_fence_history_trim();
 380     if (updated) {
 381         do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY, 0, NULL);
 382     }
 383     g_hash_table_destroy(history); /* remove what is left */
 384 }
 385 
 386 /*!
 387  * \internal
 388  * \brief Handle fence-history messages (either from API or coming in as
 389  *        broadcasts
 390  *
 391  * \param[in] msg       Request message
 392  * \param[in] output    In case of a request from the API used to craft
 393  *                      a reply from
 394  * \param[in] remote_peer
 395  * \param[in] options   call-options from the request
 396  *
 397  * \return always success as there is actully nothing that can go really wrong
 398  */
 399 int
 400 stonith_fence_history(xmlNode *msg, xmlNode **output,
     /*  */
 401                       const char *remote_peer, int options)
 402 {
 403     int rc = 0;
 404     const char *target = NULL;
 405     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER);
 406     xmlNode *out_history = NULL;
 407 
 408     if (dev) {
 409         target = crm_element_value(dev, F_STONITH_TARGET);
 410         if (target && (options & st_opt_cs_nodeid)) {
 411             int nodeid = crm_atoi(target, NULL);
 412             crm_node_t *node = crm_find_known_peer_full(nodeid, NULL, CRM_GET_PEER_ANY);
 413 
 414             if (node) {
 415                 target = node->uname;
 416             }
 417         }
 418     }
 419 
 420     if (options & st_opt_cleanup) {
 421         crm_trace("Cleaning up operations on %s in %p", target,
 422                   stonith_remote_op_list);
 423 
 424         stonith_fence_history_cleanup(target,
 425             crm_element_value(msg, F_STONITH_CALLID) != NULL);
 426     } else if (options & st_opt_broadcast) {
 427         /* there is no clear sign atm for when a history sync
 428            is done so send a notification for anything
 429            that smells like history-sync
 430          */
 431         do_stonith_notify(0, T_STONITH_NOTIFY_HISTORY_SYNCED, 0, NULL);
 432         if (crm_element_value(msg, F_STONITH_CALLID)) {
 433             /* this is coming from the stonith-API
 434             *
 435             * craft a broadcast with node's history
 436             * so that every node can merge and broadcast
 437             * what it has on top
 438             */
 439             out_history = stonith_local_history_diff(NULL, TRUE, NULL);
 440             crm_trace("Broadcasting history to peers");
 441             stonith_send_broadcast_history(out_history,
 442                                         st_opt_broadcast | st_opt_discard_reply,
 443                                         NULL);
 444         } else if (remote_peer &&
 445                    !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
 446             xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
 447                                                 msg, LOG_NEVER);
 448             GHashTable *received_history =
 449                 history?stonith_xml_history_to_list(history):NULL;
 450 
 451             /* either a broadcast created directly upon stonith-API request
 452             * or a diff as response to such a thing
 453             *
 454             * in both cases it may have a history or not
 455             * if we have differential data
 456             * merge in what we've received and stop
 457             * otherwise broadcast what we have on top
 458             * marking as differential and merge in afterwards
 459             */
 460             if (!history ||
 461                 !crm_is_true(crm_element_value(history,
 462                                                F_STONITH_DIFFERENTIAL))) {
 463                 out_history =
 464                     stonith_local_history_diff(received_history, TRUE, NULL);
 465                 if (out_history) {
 466                     crm_trace("Broadcasting history-diff to peers");
 467                     crm_xml_add(out_history, F_STONITH_DIFFERENTIAL,
 468                                 XML_BOOLEAN_TRUE);
 469                     stonith_send_broadcast_history(out_history,
 470                         st_opt_broadcast | st_opt_discard_reply,
 471                         NULL);
 472                 } else {
 473                     crm_trace("History-diff is empty - skip broadcast");
 474                 }
 475             }
 476             stonith_merge_in_history_list(received_history);
 477         } else {
 478             crm_trace("Skipping history-query-broadcast (%s%s)"
 479                       " we sent ourselves",
 480                       remote_peer?"remote-peer=":"local-ipc",
 481                       remote_peer?remote_peer:"");
 482         }
 483     } else {
 484         /* plain history request */
 485         crm_trace("Looking for operations on %s in %p", target,
 486                   stonith_remote_op_list);
 487         *output = stonith_local_history_diff(NULL, FALSE, target);
 488     }
 489     free_xml(out_history);
 490     return rc;
 491 }

/* */