failcounts.c

Go to the documentation of this file.

/*
 * Copyright 2008-2023 the Pacemaker project contributors
 *
 * This source code is licensed under the GNU Lesser General Public License
 * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
 */

#include <crm_internal.h>

#include <sys/types.h>
#include <regex.h>
#include <glib.h>

#include <crm/crm.h>
#include <crm/msg_xml.h>
#include <crm/common/xml.h>
#include <crm/common/util.h>
#include <crm/pengine/internal.h>

static gboolean
is_matched_failure(const char *rsc_id, const xmlNode *conf_op_xml,
                   const xmlNode *lrm_op_xml)
{
    gboolean matched = FALSE;
    const char *conf_op_name = NULL;
    const char *lrm_op_task = NULL;
    const char *conf_op_interval_spec = NULL;
    guint conf_op_interval_ms = 0;
    guint lrm_op_interval_ms = 0;
    const char *lrm_op_id = NULL;
    char *last_failure_key = NULL;

    if (rsc_id == NULL || conf_op_xml == NULL || lrm_op_xml == NULL) {
        return FALSE;
    }

    // Get name and interval from configured op
    conf_op_name = crm_element_value(conf_op_xml, "name");
    conf_op_interval_spec = crm_element_value(conf_op_xml,
                                              XML_LRM_ATTR_INTERVAL);
    conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

    // Get name and interval from op history entry
    lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
    crm_element_value_ms(lrm_op_xml, XML_LRM_ATTR_INTERVAL_MS,
                         &lrm_op_interval_ms);

    if ((conf_op_interval_ms != lrm_op_interval_ms)
        || !pcmk__str_eq(conf_op_name, lrm_op_task, pcmk__str_casei)) {
        return FALSE;
    }

    lrm_op_id = ID(lrm_op_xml);
    last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);

    if (pcmk__str_eq(last_failure_key, lrm_op_id, pcmk__str_casei)) {
        matched = TRUE;

    } else {
        char *expected_op_key = pcmk__op_key(rsc_id, conf_op_name,
                                                conf_op_interval_ms);

        if (pcmk__str_eq(expected_op_key, lrm_op_id, pcmk__str_casei)) {
            int rc = 0;
            int target_rc = pe__target_rc_from_xml(lrm_op_xml);

            crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
            if (rc != target_rc) {
                matched = TRUE;
            }
        }
        free(expected_op_key);
    }

    free(last_failure_key);
    return matched;
}

static gboolean
block_failure(const pcmk_node_t *node, pcmk_resource_t *rsc,
              const xmlNode *xml_op)
{
    char *xml_name = clone_strip(rsc->id);

    /* @TODO This xpath search occurs after template expansion, but it is unable
     * to properly detect on-fail in id-ref, operation meta-attributes, or
     * op_defaults, or evaluate rules.
     *
     * Also, on-fail defaults to block (in unpack_operation()) for stop actions
     * when stonith is disabled.
     *
     * Ideally, we'd unpack the operation before this point, and pass in a
     * meta-attributes table that takes all that into consideration.
     */
    char *xpath = crm_strdup_printf("//" XML_CIB_TAG_RESOURCE
                                    "[@" XML_ATTR_ID "='%s']"
                                    "//" XML_ATTR_OP
                                    "[@" XML_OP_ATTR_ON_FAIL "='block']",
                                    xml_name);

    xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
    gboolean should_block = FALSE;

    free(xpath);

    if (xpathObj) {
        int max = numXpathResults(xpathObj);
        int lpc = 0;

        for (lpc = 0; lpc < max; lpc++) {
            xmlNode *pref = getXpathResult(xpathObj, lpc);

            if (xml_op) {
                should_block = is_matched_failure(xml_name, pref, xml_op);
                if (should_block) {
                    break;
                }

            } else {
                const char *conf_op_name = NULL;
                const char *conf_op_interval_spec = NULL;
                guint conf_op_interval_ms = 0;
                char *lrm_op_xpath = NULL;
                xmlXPathObject *lrm_op_xpathObj = NULL;

                // Get name and interval from configured op
                conf_op_name = crm_element_value(pref, "name");
                conf_op_interval_spec = crm_element_value(pref, XML_LRM_ATTR_INTERVAL);
                conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);

#define XPATH_FMT "//" XML_CIB_TAG_STATE "[@" XML_ATTR_UNAME "='%s']"       \
                  "//" XML_LRM_TAG_RESOURCE "[@" XML_ATTR_ID "='%s']"       \
                  "/" XML_LRM_TAG_RSC_OP "[@" XML_LRM_ATTR_TASK "='%s']"    \
                  "[@" XML_LRM_ATTR_INTERVAL "='%u']"

                lrm_op_xpath = crm_strdup_printf(XPATH_FMT,
                                                 node->details->uname, xml_name,
                                                 conf_op_name,
                                                 conf_op_interval_ms);
                lrm_op_xpathObj = xpath_search(rsc->cluster->input, lrm_op_xpath);

                free(lrm_op_xpath);

                if (lrm_op_xpathObj) {
                    int max2 = numXpathResults(lrm_op_xpathObj);
                    int lpc2 = 0;

                    for (lpc2 = 0; lpc2 < max2; lpc2++) {
                        xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
                                                             lpc2);

                        should_block = is_matched_failure(xml_name, pref,
                                                          lrm_op_xml);
                        if (should_block) {
                            break;
                        }
                    }
                }
                freeXpathObject(lrm_op_xpathObj);

                if (should_block) {
                    break;
                }
            }
        }
    }

    free(xml_name);
    freeXpathObject(xpathObj);

    return should_block;
}

static inline char *
rsc_fail_name(const pcmk_resource_t *rsc)
{
    const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);

    return pcmk_is_set(rsc->flags, pcmk_rsc_unique)? strdup(name) : clone_strip(name);
}

static int
generate_fail_regex(const char *prefix, const char *rsc_name,
                    gboolean is_legacy, gboolean is_unique, regex_t *re)
{
    char *pattern;

    /* @COMPAT DC < 1.1.17: Fail counts used to be per-resource rather than
     * per-operation.
     */
    const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");

    /* Ignore instance numbers for anything other than globally unique clones.
     * Anonymous clone fail counts could contain an instance number if the
     * clone was initially unique, failed, then was converted to anonymous.
     * @COMPAT Also, before 1.1.8, anonymous clone fail counts always contained
     * clone instance numbers.
     */
    const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");

    pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
                                instance_pattern, op_pattern);
    if (regcomp(re, pattern, REG_EXTENDED|REG_NOSUB) != 0) {
        free(pattern);
        return EINVAL;
    }

    free(pattern);
    return pcmk_rc_ok;
}

static int
generate_fail_regexes(const pcmk_resource_t *rsc,
                      regex_t *failcount_re, regex_t *lastfailure_re)
{
    int rc = pcmk_rc_ok;
    char *rsc_name = rsc_fail_name(rsc);
    const char *version = crm_element_value(rsc->cluster->input,
                                            XML_ATTR_CRM_VERSION);

    // @COMPAT Pacemaker <= 1.1.16 used a single fail count per resource
    gboolean is_legacy = (compare_version(version, "3.0.13") < 0);

    if (generate_fail_regex(PCMK__FAIL_COUNT_PREFIX, rsc_name, is_legacy,
                            pcmk_is_set(rsc->flags, pcmk_rsc_unique),
                            failcount_re) != pcmk_rc_ok) {
        rc = EINVAL;

    } else if (generate_fail_regex(PCMK__LAST_FAILURE_PREFIX, rsc_name,
                                   is_legacy,
                                   pcmk_is_set(rsc->flags, pcmk_rsc_unique),
                                   lastfailure_re) != pcmk_rc_ok) {
        rc = EINVAL;
        regfree(failcount_re);
    }

    free(rsc_name);
    return rc;
}

// Data for fail-count-related iterators
struct failcount_data {
    const pcmk_node_t *node;// Node to check for fail count
    pcmk_resource_t *rsc;     // Resource to check for fail count
    uint32_t flags;         // Fail count flags
    const xmlNode *xml_op;  // History entry for expiration purposes (or NULL)
    regex_t failcount_re;   // Fail count regular expression to match
    regex_t lastfailure_re; // Last failure regular expression to match
    int failcount;          // Fail count so far
    time_t last_failure;    // Time of most recent failure so far
};

static void
update_failcount_for_attr(gpointer key, gpointer value, gpointer user_data)
{
    struct failcount_data *fc_data = user_data;

    // If this is a matching fail count attribute, update fail count
    if (regexec(&(fc_data->failcount_re), (const char *) key, 0, NULL, 0) == 0) {
        fc_data->failcount = pcmk__add_scores(fc_data->failcount,
                                              char2score(value));
        pe_rsc_trace(fc_data->rsc, "Added %s (%s) to %s fail count (now %s)",
                     (const char *) key, (const char *) value, fc_data->rsc->id,
                     pcmk_readable_score(fc_data->failcount));
        return;
    }

    // If this is a matching last failure attribute, update last failure
    if (regexec(&(fc_data->lastfailure_re), (const char *) key, 0, NULL,
                0) == 0) {
        long long last_ll;

        if (pcmk__scan_ll(value, &last_ll, 0LL) == pcmk_rc_ok) {
            fc_data->last_failure = (time_t) QB_MAX(fc_data->last_failure,
                                                    last_ll);
        }
    }
}

static void
update_failcount_for_filler(gpointer data, gpointer user_data)
{
    pcmk_resource_t *filler = data;
    struct failcount_data *fc_data = user_data;
    time_t filler_last_failure = 0;

    fc_data->failcount += pe_get_failcount(fc_data->node, filler,
                                           &filler_last_failure, fc_data->flags,
                                           fc_data->xml_op);
    fc_data->last_failure = QB_MAX(fc_data->last_failure, filler_last_failure);
}

int
pe_get_failcount(const pcmk_node_t *node, pcmk_resource_t *rsc,
                 time_t *last_failure, uint32_t flags, const xmlNode *xml_op)
{
    struct failcount_data fc_data = {
        .node = node,
        .rsc = rsc,
        .flags = flags,
        .xml_op = xml_op,
        .failcount = 0,
        .last_failure = (time_t) 0,
    };

    // Calculate resource failcount as sum of all matching operation failcounts
    CRM_CHECK(generate_fail_regexes(rsc, &fc_data.failcount_re,
                                    &fc_data.lastfailure_re) == pcmk_rc_ok,
              return 0);
    g_hash_table_foreach(node->details->attrs, update_failcount_for_attr,
                         &fc_data);
    regfree(&(fc_data.failcount_re));
    regfree(&(fc_data.lastfailure_re));

    // If failure blocks the resource, disregard any failure timeout
    if ((fc_data.failcount > 0) && (rsc->failure_timeout > 0)
        && block_failure(node, rsc, xml_op)) {

        pe_warn("Ignoring failure timeout %d for %s "
                "because it conflicts with on-fail=block",
                rsc->failure_timeout, rsc->id);
        rsc->failure_timeout = 0;
    }

    // If all failures have expired, ignore fail count
    if (pcmk_is_set(flags, pcmk__fc_effective) && (fc_data.failcount > 0)
        && (fc_data.last_failure > 0) && (rsc->failure_timeout != 0)) {

        time_t now = get_effective_time(rsc->cluster);

        if (now > (fc_data.last_failure + rsc->failure_timeout)) {
            pe_rsc_debug(rsc, "Failcount for %s on %s expired after %ds",
                         rsc->id, pe__node_name(node), rsc->failure_timeout);
            fc_data.failcount = 0;
        }
    }

    /* Add the fail count of any filler resources, except that we never want the
     * fail counts of a bundle container's fillers to count towards the
     * container's fail count.
     *
     * Most importantly, a Pacemaker Remote connection to a bundle container
     * is a filler of the container, but can reside on a different node than the
     * container itself. Counting its fail count on its node towards the
     * container's fail count on that node could lead to attempting to stop the
     * container on the wrong node.
     */
    if (pcmk_is_set(flags, pcmk__fc_fillers) && (rsc->fillers != NULL)
        && !pe_rsc_is_bundled(rsc)) {

        g_list_foreach(rsc->fillers, update_failcount_for_filler, &fc_data);
        if (fc_data.failcount > 0) {
            pe_rsc_info(rsc,
                        "Container %s and the resources within it "
                        "have failed %s time%s on %s",
                        rsc->id, pcmk_readable_score(fc_data.failcount),
                        pcmk__plural_s(fc_data.failcount), pe__node_name(node));
        }

    } else if (fc_data.failcount > 0) {
        pe_rsc_info(rsc, "%s has failed %s time%s on %s",
                    rsc->id, pcmk_readable_score(fc_data.failcount),
                    pcmk__plural_s(fc_data.failcount), pe__node_name(node));
    }

    if (last_failure != NULL) {
        if ((fc_data.failcount > 0) && (fc_data.last_failure > 0)) {
            *last_failure = fc_data.last_failure;
        } else  {
            *last_failure = 0;
        }
    }
    return fc_data.failcount;
}

pcmk_action_t *
pe__clear_failcount(pcmk_resource_t *rsc, const pcmk_node_t *node,
                    const char *reason, pcmk_scheduler_t *scheduler)
{
    char *key = NULL;
    pcmk_action_t *clear = NULL;

    CRM_CHECK(rsc && node && reason && scheduler, return NULL);

    key = pcmk__op_key(rsc->id, PCMK_ACTION_CLEAR_FAILCOUNT, 0);
    clear = custom_action(rsc, key, PCMK_ACTION_CLEAR_FAILCOUNT, node, FALSE,
                          scheduler);
    add_hash_param(clear->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
    crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
               rsc->id, pe__node_name(node), reason, clear->uuid);
    return clear;
}