This source file includes following definitions.
- is_matched_failure
- block_failure
- rsc_fail_name
- generate_fail_regex
- generate_fail_regexes
- update_failcount_for_attr
- update_failcount_for_filler
- pe_get_failcount
- pe__clear_failcount
1
2
3
4
5
6
7
8 #include <crm_internal.h>
9
10 #include <sys/types.h>
11 #include <regex.h>
12 #include <glib.h>
13
14 #include <crm/crm.h>
15 #include <crm/common/xml.h>
16 #include <crm/common/util.h>
17 #include <crm/pengine/internal.h>
18
19 static gboolean
20 is_matched_failure(const char *rsc_id, const xmlNode *conf_op_xml,
21 const xmlNode *lrm_op_xml)
22 {
23 gboolean matched = FALSE;
24 const char *conf_op_name = NULL;
25 const char *lrm_op_task = NULL;
26 const char *conf_op_interval_spec = NULL;
27 guint conf_op_interval_ms = 0;
28 guint lrm_op_interval_ms = 0;
29 const char *lrm_op_id = NULL;
30 char *last_failure_key = NULL;
31
32 if (rsc_id == NULL || conf_op_xml == NULL || lrm_op_xml == NULL) {
33 return FALSE;
34 }
35
36
37 conf_op_name = crm_element_value(conf_op_xml, PCMK_XA_NAME);
38 conf_op_interval_spec = crm_element_value(conf_op_xml, PCMK_META_INTERVAL);
39 pcmk_parse_interval_spec(conf_op_interval_spec, &conf_op_interval_ms);
40
41
42 lrm_op_task = crm_element_value(lrm_op_xml, PCMK_XA_OPERATION);
43 crm_element_value_ms(lrm_op_xml, PCMK_META_INTERVAL, &lrm_op_interval_ms);
44
45 if ((conf_op_interval_ms != lrm_op_interval_ms)
46 || !pcmk__str_eq(conf_op_name, lrm_op_task, pcmk__str_casei)) {
47 return FALSE;
48 }
49
50 lrm_op_id = pcmk__xe_id(lrm_op_xml);
51 last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
52
53 if (pcmk__str_eq(last_failure_key, lrm_op_id, pcmk__str_casei)) {
54 matched = TRUE;
55
56 } else {
57 char *expected_op_key = pcmk__op_key(rsc_id, conf_op_name,
58 conf_op_interval_ms);
59
60 if (pcmk__str_eq(expected_op_key, lrm_op_id, pcmk__str_casei)) {
61 int rc = 0;
62 int target_rc = pe__target_rc_from_xml(lrm_op_xml);
63
64 crm_element_value_int(lrm_op_xml, PCMK__XA_RC_CODE, &rc);
65 if (rc != target_rc) {
66 matched = TRUE;
67 }
68 }
69 free(expected_op_key);
70 }
71
72 free(last_failure_key);
73 return matched;
74 }
75
76 static gboolean
77 block_failure(const pcmk_node_t *node, pcmk_resource_t *rsc,
78 const xmlNode *xml_op)
79 {
80 char *xml_name = clone_strip(rsc->id);
81
82
83
84
85
86
87
88
89
90
91
92 char *xpath = crm_strdup_printf("//" PCMK_XE_PRIMITIVE
93 "[@" PCMK_XA_ID "='%s']"
94 "//" PCMK_XE_OP
95 "[@" PCMK_META_ON_FAIL
96 "='" PCMK_VALUE_BLOCK "']",
97 xml_name);
98
99 xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
100 gboolean should_block = FALSE;
101
102 free(xpath);
103
104 if (xpathObj) {
105 int max = numXpathResults(xpathObj);
106 int lpc = 0;
107
108 for (lpc = 0; lpc < max; lpc++) {
109 xmlNode *pref = getXpathResult(xpathObj, lpc);
110
111 if (xml_op) {
112 should_block = is_matched_failure(xml_name, pref, xml_op);
113 if (should_block) {
114 break;
115 }
116
117 } else {
118 const char *conf_op_name = NULL;
119 const char *conf_op_interval_spec = NULL;
120 guint conf_op_interval_ms = 0;
121 char *lrm_op_xpath = NULL;
122 xmlXPathObject *lrm_op_xpathObj = NULL;
123
124
125 conf_op_name = crm_element_value(pref, PCMK_XA_NAME);
126 conf_op_interval_spec = crm_element_value(pref,
127 PCMK_META_INTERVAL);
128 pcmk_parse_interval_spec(conf_op_interval_spec,
129 &conf_op_interval_ms);
130
131 #define XPATH_FMT "//" PCMK__XE_NODE_STATE "[@" PCMK_XA_UNAME "='%s']" \
132 "//" PCMK__XE_LRM_RESOURCE "[@" PCMK_XA_ID "='%s']" \
133 "/" PCMK__XE_LRM_RSC_OP "[@" PCMK_XA_OPERATION "='%s']" \
134 "[@" PCMK_META_INTERVAL "='%u']"
135
136 lrm_op_xpath = crm_strdup_printf(XPATH_FMT,
137 node->details->uname, xml_name,
138 conf_op_name,
139 conf_op_interval_ms);
140 lrm_op_xpathObj = xpath_search(rsc->cluster->input, lrm_op_xpath);
141
142 free(lrm_op_xpath);
143
144 if (lrm_op_xpathObj) {
145 int max2 = numXpathResults(lrm_op_xpathObj);
146 int lpc2 = 0;
147
148 for (lpc2 = 0; lpc2 < max2; lpc2++) {
149 xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
150 lpc2);
151
152 should_block = is_matched_failure(xml_name, pref,
153 lrm_op_xml);
154 if (should_block) {
155 break;
156 }
157 }
158 }
159 freeXpathObject(lrm_op_xpathObj);
160
161 if (should_block) {
162 break;
163 }
164 }
165 }
166 }
167
168 free(xml_name);
169 freeXpathObject(xpathObj);
170
171 return should_block;
172 }
173
174
175
176
177
178
179
180
181
182
183 static inline char *
184 rsc_fail_name(const pcmk_resource_t *rsc)
185 {
186 const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
187
188 return pcmk_is_set(rsc->flags, pcmk_rsc_unique)? strdup(name) : clone_strip(name);
189 }
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205 static int
206 generate_fail_regex(const char *prefix, const char *rsc_name,
207 gboolean is_legacy, gboolean is_unique, regex_t *re)
208 {
209 char *pattern;
210
211
212
213
214 const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");
215
216
217
218
219
220
221
222 const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");
223
224 pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
225 instance_pattern, op_pattern);
226 if (regcomp(re, pattern, REG_EXTENDED|REG_NOSUB) != 0) {
227 free(pattern);
228 return EINVAL;
229 }
230
231 free(pattern);
232 return pcmk_rc_ok;
233 }
234
235
236
237
238
239
240
241
242
243
244
245
246
247 static int
248 generate_fail_regexes(const pcmk_resource_t *rsc,
249 regex_t *failcount_re, regex_t *lastfailure_re)
250 {
251 int rc = pcmk_rc_ok;
252 char *rsc_name = rsc_fail_name(rsc);
253 const char *version = crm_element_value(rsc->cluster->input,
254 PCMK_XA_CRM_FEATURE_SET);
255
256
257 gboolean is_legacy = (compare_version(version, "3.0.13") < 0);
258
259 if (generate_fail_regex(PCMK__FAIL_COUNT_PREFIX, rsc_name, is_legacy,
260 pcmk_is_set(rsc->flags, pcmk_rsc_unique),
261 failcount_re) != pcmk_rc_ok) {
262 rc = EINVAL;
263
264 } else if (generate_fail_regex(PCMK__LAST_FAILURE_PREFIX, rsc_name,
265 is_legacy,
266 pcmk_is_set(rsc->flags, pcmk_rsc_unique),
267 lastfailure_re) != pcmk_rc_ok) {
268 rc = EINVAL;
269 regfree(failcount_re);
270 }
271
272 free(rsc_name);
273 return rc;
274 }
275
276
277 struct failcount_data {
278 const pcmk_node_t *node;
279 pcmk_resource_t *rsc;
280 uint32_t flags;
281 const xmlNode *xml_op;
282 regex_t failcount_re;
283 regex_t lastfailure_re;
284 int failcount;
285 time_t last_failure;
286 };
287
288
289
290
291
292
293
294
295
296 static void
297 update_failcount_for_attr(gpointer key, gpointer value, gpointer user_data)
298 {
299 struct failcount_data *fc_data = user_data;
300
301
302 if (regexec(&(fc_data->failcount_re), (const char *) key, 0, NULL, 0) == 0) {
303 int score = 0;
304 int rc = pcmk_parse_score(value, &score, 0);
305
306 if (rc != pcmk_rc_ok) {
307 crm_warn("Ignoring %s for %s "
308 "because '%s' is not a valid fail count: %s",
309 (const char *) key, pcmk__node_name(fc_data->node),
310 value, pcmk_rc_str(rc));
311 return;
312 }
313 fc_data->failcount = pcmk__add_scores(fc_data->failcount, score);
314 pcmk__rsc_trace(fc_data->rsc, "Added %s (%s) to %s fail count (now %s)",
315 (const char *) key, (const char *) value,
316 fc_data->rsc->id,
317 pcmk_readable_score(fc_data->failcount));
318 return;
319 }
320
321
322 if (regexec(&(fc_data->lastfailure_re), (const char *) key, 0, NULL,
323 0) == 0) {
324 long long last_ll;
325 int rc = pcmk__scan_ll(value, &last_ll, 0LL);
326
327 if (rc != pcmk_rc_ok) {
328 crm_info("Ignoring invalid value '%s' for %s: %s",
329 (const char *) value, (const char *) key, pcmk_rc_str(rc));
330 return;
331 }
332 fc_data->last_failure = (time_t) QB_MAX(fc_data->last_failure, last_ll);
333 }
334 }
335
336
337
338
339
340
341
342
343 static void
344 update_failcount_for_filler(gpointer data, gpointer user_data)
345 {
346 pcmk_resource_t *filler = data;
347 struct failcount_data *fc_data = user_data;
348 time_t filler_last_failure = 0;
349
350 fc_data->failcount += pe_get_failcount(fc_data->node, filler,
351 &filler_last_failure, fc_data->flags,
352 fc_data->xml_op);
353 fc_data->last_failure = QB_MAX(fc_data->last_failure, filler_last_failure);
354 }
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372 int
373 pe_get_failcount(const pcmk_node_t *node, pcmk_resource_t *rsc,
374 time_t *last_failure, uint32_t flags, const xmlNode *xml_op)
375 {
376 struct failcount_data fc_data = {
377 .node = node,
378 .rsc = rsc,
379 .flags = flags,
380 .xml_op = xml_op,
381 .failcount = 0,
382 .last_failure = (time_t) 0,
383 };
384
385
386 CRM_CHECK(generate_fail_regexes(rsc, &fc_data.failcount_re,
387 &fc_data.lastfailure_re) == pcmk_rc_ok,
388 return 0);
389 g_hash_table_foreach(node->details->attrs, update_failcount_for_attr,
390 &fc_data);
391 regfree(&(fc_data.failcount_re));
392 regfree(&(fc_data.lastfailure_re));
393
394
395 if ((fc_data.failcount > 0) && (rsc->failure_timeout > 0)
396 && block_failure(node, rsc, xml_op)) {
397
398 pcmk__config_warn("Ignoring failure timeout %d for %s "
399 "because it conflicts with "
400 PCMK_META_ON_FAIL "=" PCMK_VALUE_BLOCK,
401 rsc->failure_timeout, rsc->id);
402 rsc->failure_timeout = 0;
403 }
404
405
406 if (pcmk_is_set(flags, pcmk__fc_effective) && (fc_data.failcount > 0)
407 && (fc_data.last_failure > 0) && (rsc->failure_timeout != 0)) {
408
409 time_t now = get_effective_time(rsc->cluster);
410
411 if (now > (fc_data.last_failure + rsc->failure_timeout)) {
412 pcmk__rsc_debug(rsc, "Failcount for %s on %s expired after %ds",
413 rsc->id, pcmk__node_name(node),
414 rsc->failure_timeout);
415 fc_data.failcount = 0;
416 }
417 }
418
419
420
421
422
423
424
425
426
427
428
429 if (pcmk_is_set(flags, pcmk__fc_fillers) && (rsc->fillers != NULL)
430 && !pcmk__is_bundled(rsc)) {
431
432 g_list_foreach(rsc->fillers, update_failcount_for_filler, &fc_data);
433 if (fc_data.failcount > 0) {
434 pcmk__rsc_info(rsc,
435 "Container %s and the resources within it "
436 "have failed %s time%s on %s",
437 rsc->id, pcmk_readable_score(fc_data.failcount),
438 pcmk__plural_s(fc_data.failcount),
439 pcmk__node_name(node));
440 }
441
442 } else if (fc_data.failcount > 0) {
443 pcmk__rsc_info(rsc, "%s has failed %s time%s on %s",
444 rsc->id, pcmk_readable_score(fc_data.failcount),
445 pcmk__plural_s(fc_data.failcount),
446 pcmk__node_name(node));
447 }
448
449 if (last_failure != NULL) {
450 if ((fc_data.failcount > 0) && (fc_data.last_failure > 0)) {
451 *last_failure = fc_data.last_failure;
452 } else {
453 *last_failure = 0;
454 }
455 }
456 return fc_data.failcount;
457 }
458
459
460
461
462
463
464
465
466
467
468
469 pcmk_action_t *
470 pe__clear_failcount(pcmk_resource_t *rsc, const pcmk_node_t *node,
471 const char *reason, pcmk_scheduler_t *scheduler)
472 {
473 char *key = NULL;
474 pcmk_action_t *clear = NULL;
475
476 CRM_CHECK(rsc && node && reason && scheduler, return NULL);
477
478 key = pcmk__op_key(rsc->id, PCMK_ACTION_CLEAR_FAILCOUNT, 0);
479 clear = custom_action(rsc, key, PCMK_ACTION_CLEAR_FAILCOUNT, node, FALSE,
480 scheduler);
481 pcmk__insert_meta(clear, PCMK__META_OP_NO_WAIT, PCMK_VALUE_TRUE);
482 crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
483 rsc->id, pcmk__node_name(node), reason, clear->uuid);
484 return clear;
485 }