This source file includes following definitions.
- is_matched_failure
- block_failure
- rsc_fail_name
- generate_fail_regex
- generate_fail_regexes
- pe_get_failcount
- pe__clear_failcount
1
2
3
4
5
6
7
8 #include <crm_internal.h>
9
10 #include <sys/types.h>
11 #include <regex.h>
12 #include <glib.h>
13
14 #include <crm/crm.h>
15 #include <crm/msg_xml.h>
16 #include <crm/common/xml.h>
17 #include <crm/common/util.h>
18 #include <crm/pengine/internal.h>
19
20 static gboolean
21 is_matched_failure(const char *rsc_id, xmlNode *conf_op_xml,
22 xmlNode *lrm_op_xml)
23 {
24 gboolean matched = FALSE;
25 const char *conf_op_name = NULL;
26 const char *lrm_op_task = NULL;
27 const char *conf_op_interval_spec = NULL;
28 guint conf_op_interval_ms = 0;
29 guint lrm_op_interval_ms = 0;
30 const char *lrm_op_id = NULL;
31 char *last_failure_key = NULL;
32
33 if (rsc_id == NULL || conf_op_xml == NULL || lrm_op_xml == NULL) {
34 return FALSE;
35 }
36
37
38 conf_op_name = crm_element_value(conf_op_xml, "name");
39 conf_op_interval_spec = crm_element_value(conf_op_xml,
40 XML_LRM_ATTR_INTERVAL);
41 conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);
42
43
44 lrm_op_task = crm_element_value(lrm_op_xml, XML_LRM_ATTR_TASK);
45 crm_element_value_ms(lrm_op_xml, XML_LRM_ATTR_INTERVAL_MS,
46 &lrm_op_interval_ms);
47
48 if ((conf_op_interval_ms != lrm_op_interval_ms)
49 || !pcmk__str_eq(conf_op_name, lrm_op_task, pcmk__str_casei)) {
50 return FALSE;
51 }
52
53 lrm_op_id = ID(lrm_op_xml);
54 last_failure_key = pcmk__op_key(rsc_id, "last_failure", 0);
55
56 if (pcmk__str_eq(last_failure_key, lrm_op_id, pcmk__str_casei)) {
57 matched = TRUE;
58
59 } else {
60 char *expected_op_key = pcmk__op_key(rsc_id, conf_op_name,
61 conf_op_interval_ms);
62
63 if (pcmk__str_eq(expected_op_key, lrm_op_id, pcmk__str_casei)) {
64 int rc = 0;
65 int target_rc = pe__target_rc_from_xml(lrm_op_xml);
66
67 crm_element_value_int(lrm_op_xml, XML_LRM_ATTR_RC, &rc);
68 if (rc != target_rc) {
69 matched = TRUE;
70 }
71 }
72 free(expected_op_key);
73 }
74
75 free(last_failure_key);
76 return matched;
77 }
78
79 static gboolean
80 block_failure(pe_node_t *node, pe_resource_t *rsc, xmlNode *xml_op,
81 pe_working_set_t *data_set)
82 {
83 char *xml_name = clone_strip(rsc->id);
84
85
86
87
88
89
90
91
92
93
94
95 char *xpath = crm_strdup_printf("//primitive[@id='%s']//op[@on-fail='block']",
96 xml_name);
97
98 xmlXPathObject *xpathObj = xpath_search(rsc->xml, xpath);
99 gboolean should_block = FALSE;
100
101 free(xpath);
102
103 if (xpathObj) {
104 int max = numXpathResults(xpathObj);
105 int lpc = 0;
106
107 for (lpc = 0; lpc < max; lpc++) {
108 xmlNode *pref = getXpathResult(xpathObj, lpc);
109
110 if (xml_op) {
111 should_block = is_matched_failure(xml_name, pref, xml_op);
112 if (should_block) {
113 break;
114 }
115
116 } else {
117 const char *conf_op_name = NULL;
118 const char *conf_op_interval_spec = NULL;
119 guint conf_op_interval_ms = 0;
120 char *lrm_op_xpath = NULL;
121 xmlXPathObject *lrm_op_xpathObj = NULL;
122
123
124 conf_op_name = crm_element_value(pref, "name");
125 conf_op_interval_spec = crm_element_value(pref, XML_LRM_ATTR_INTERVAL);
126 conf_op_interval_ms = crm_parse_interval_spec(conf_op_interval_spec);
127
128 lrm_op_xpath = crm_strdup_printf("//node_state[@uname='%s']"
129 "//lrm_resource[@id='%s']"
130 "/lrm_rsc_op[@operation='%s'][@interval='%u']",
131 node->details->uname, xml_name,
132 conf_op_name, conf_op_interval_ms);
133 lrm_op_xpathObj = xpath_search(data_set->input, lrm_op_xpath);
134
135 free(lrm_op_xpath);
136
137 if (lrm_op_xpathObj) {
138 int max2 = numXpathResults(lrm_op_xpathObj);
139 int lpc2 = 0;
140
141 for (lpc2 = 0; lpc2 < max2; lpc2++) {
142 xmlNode *lrm_op_xml = getXpathResult(lrm_op_xpathObj,
143 lpc2);
144
145 should_block = is_matched_failure(xml_name, pref,
146 lrm_op_xml);
147 if (should_block) {
148 break;
149 }
150 }
151 }
152 freeXpathObject(lrm_op_xpathObj);
153
154 if (should_block) {
155 break;
156 }
157 }
158 }
159 }
160
161 free(xml_name);
162 freeXpathObject(xpathObj);
163
164 return should_block;
165 }
166
167
168
169
170
171
172
173
174
175
176 static inline char *
177 rsc_fail_name(pe_resource_t *rsc)
178 {
179 const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
180
181 return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
182 }
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197 static void
198 generate_fail_regex(const char *prefix, const char *rsc_name,
199 gboolean is_legacy, gboolean is_unique, regex_t *re)
200 {
201 char *pattern;
202
203
204
205
206 const char *op_pattern = (is_legacy? "" : "#.+_[0-9]+");
207
208
209
210
211
212
213
214 const char *instance_pattern = (is_unique? "" : "(:[0-9]+)?");
215
216 pattern = crm_strdup_printf("^%s-%s%s%s$", prefix, rsc_name,
217 instance_pattern, op_pattern);
218 CRM_LOG_ASSERT(regcomp(re, pattern, REG_EXTENDED|REG_NOSUB) == 0);
219 free(pattern);
220 }
221
222
223
224
225
226
227
228
229
230
231
232
233 static void
234 generate_fail_regexes(pe_resource_t *rsc, pe_working_set_t *data_set,
235 regex_t *failcount_re, regex_t *lastfailure_re)
236 {
237 char *rsc_name = rsc_fail_name(rsc);
238 const char *version = crm_element_value(data_set->input, XML_ATTR_CRM_VERSION);
239 gboolean is_legacy = (compare_version(version, "3.0.13") < 0);
240
241 generate_fail_regex(PCMK__FAIL_COUNT_PREFIX, rsc_name, is_legacy,
242 pcmk_is_set(rsc->flags, pe_rsc_unique), failcount_re);
243
244 generate_fail_regex(PCMK__LAST_FAILURE_PREFIX, rsc_name, is_legacy,
245 pcmk_is_set(rsc->flags, pe_rsc_unique), lastfailure_re);
246
247 free(rsc_name);
248 }
249
250 int
251 pe_get_failcount(pe_node_t *node, pe_resource_t *rsc, time_t *last_failure,
252 uint32_t flags, xmlNode *xml_op, pe_working_set_t *data_set)
253 {
254 char *key = NULL;
255 const char *value = NULL;
256 regex_t failcount_re, lastfailure_re;
257 int failcount = 0;
258 time_t last = 0;
259 GHashTableIter iter;
260
261 generate_fail_regexes(rsc, data_set, &failcount_re, &lastfailure_re);
262
263
264 g_hash_table_iter_init(&iter, node->details->attrs);
265 while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
266 if (regexec(&failcount_re, key, 0, NULL, 0) == 0) {
267 failcount = pe__add_scores(failcount, char2score(value));
268 } else if (regexec(&lastfailure_re, key, 0, NULL, 0) == 0) {
269 long long last_ll;
270
271 if (pcmk__scan_ll(value, &last_ll, 0LL) == pcmk_rc_ok) {
272 last = (time_t) QB_MAX(last, last_ll);
273 }
274 }
275 }
276
277 regfree(&failcount_re);
278 regfree(&lastfailure_re);
279
280 if ((failcount > 0) && (last > 0) && (last_failure != NULL)) {
281 *last_failure = last;
282 }
283
284
285 if ((failcount > 0) && rsc->failure_timeout
286 && block_failure(node, rsc, xml_op, data_set)) {
287
288 pe_warn("Ignoring failure timeout %d for %s because it conflicts with on-fail=block",
289 rsc->failure_timeout, rsc->id);
290 rsc->failure_timeout = 0;
291 }
292
293
294 if (pcmk_is_set(flags, pe_fc_effective) && (failcount > 0) && (last > 0)
295 && rsc->failure_timeout) {
296
297 time_t now = get_effective_time(data_set);
298
299 if (now > (last + rsc->failure_timeout)) {
300 crm_debug("Failcount for %s on %s expired after %ds",
301 rsc->id, node->details->uname, rsc->failure_timeout);
302 failcount = 0;
303 }
304 }
305
306
307
308
309
310
311
312
313
314
315
316 if (pcmk_is_set(flags, pe_fc_fillers) && rsc->fillers
317 && !pe_rsc_is_bundled(rsc)) {
318
319 GList *gIter = NULL;
320
321 for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
322 pe_resource_t *filler = (pe_resource_t *) gIter->data;
323 time_t filler_last_failure = 0;
324
325 failcount += pe_get_failcount(node, filler, &filler_last_failure,
326 flags, xml_op, data_set);
327
328 if (last_failure && filler_last_failure > *last_failure) {
329 *last_failure = filler_last_failure;
330 }
331 }
332
333 if (failcount > 0) {
334 char *score = score2char(failcount);
335
336 crm_info("Container %s and the resources within it have failed %s times on %s",
337 rsc->id, score, node->details->uname);
338 free(score);
339 }
340
341 } else if (failcount > 0) {
342 char *score = score2char(failcount);
343
344 crm_info("%s has failed %s times on %s",
345 rsc->id, score, node->details->uname);
346 free(score);
347 }
348
349
350 return failcount;
351 }
352
353
354
355
356
357
358
359
360
361
362
363 pe_action_t *
364 pe__clear_failcount(pe_resource_t *rsc, pe_node_t *node,
365 const char *reason, pe_working_set_t *data_set)
366 {
367 char *key = NULL;
368 pe_action_t *clear = NULL;
369
370 CRM_CHECK(rsc && node && reason && data_set, return NULL);
371
372 key = pcmk__op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
373 clear = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE,
374 data_set);
375 add_hash_param(clear->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
376 crm_notice("Clearing failure of %s on %s because %s " CRM_XS " %s",
377 rsc->id, node->details->uname, reason, clear->uuid);
378 return clear;
379 }