This source file includes following definitions.
- stonith_send_broadcast_history
- stonith_remove_history_entry
- stonith_fence_history_cleanup
- op_time_sort
- stonith_fence_history_trim
- stonith_xml_history_to_list
- stonith_local_history_diff_and_merge
- stonith_local_history
- stonith_fence_history
1
2
3
4
5
6
7
8
9
10 #include <crm_internal.h>
11
12 #include <stdio.h>
13 #include <unistd.h>
14 #include <stdlib.h>
15
16 #include <crm/crm.h>
17 #include <crm/msg_xml.h>
18 #include <crm/common/ipc.h>
19 #include <crm/common/ipc_internal.h>
20 #include <crm/cluster/internal.h>
21
22 #include <crm/stonith-ng.h>
23 #include <crm/fencing/internal.h>
24 #include <crm/common/xml.h>
25 #include <crm/common/xml_internal.h>
26
27 #include <pacemaker-fenced.h>
28
29 #define MAX_STONITH_HISTORY 500
30
31
32
33
34
35
36
37
38
39
40 static void
41 stonith_send_broadcast_history(xmlNode *history,
42 int callopts,
43 const char *target)
44 {
45 xmlNode *bcast = create_xml_node(NULL, "stonith_command");
46 xmlNode *data = create_xml_node(NULL, __func__);
47
48 if (target) {
49 crm_xml_add(data, F_STONITH_TARGET, target);
50 }
51 crm_xml_add(bcast, F_TYPE, T_STONITH_NG);
52 crm_xml_add(bcast, F_SUBTYPE, "broadcast");
53 crm_xml_add(bcast, F_STONITH_OPERATION, STONITH_OP_FENCE_HISTORY);
54 crm_xml_add_int(bcast, F_STONITH_CALLOPTS, callopts);
55 if (history) {
56 add_node_copy(data, history);
57 }
58 add_message_xml(bcast, F_STONITH_CALLDATA, data);
59 send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
60
61 free_xml(data);
62 free_xml(bcast);
63 }
64
65 static gboolean
66 stonith_remove_history_entry (gpointer key,
67 gpointer value,
68 gpointer user_data)
69 {
70 remote_fencing_op_t *op = value;
71 const char *target = (const char *) user_data;
72
73 if ((op->state == st_failed) || (op->state == st_done)) {
74 if ((target) && (strcmp(op->target, target) != 0)) {
75 return FALSE;
76 }
77 return TRUE;
78 }
79
80 return FALSE;
81 }
82
83
84
85
86
87
88
89
90 static void
91 stonith_fence_history_cleanup(const char *target,
92 gboolean broadcast)
93 {
94 if (broadcast) {
95 stonith_send_broadcast_history(NULL,
96 st_opt_cleanup | st_opt_discard_reply,
97 target);
98
99 } else if (stonith_remote_op_list) {
100 g_hash_table_foreach_remove(stonith_remote_op_list,
101 stonith_remove_history_entry,
102 (gpointer) target);
103 fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
104 }
105 }
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137 static int
138 op_time_sort(const void *a_voidp, const void *b_voidp)
139 {
140 const remote_fencing_op_t **a = (const remote_fencing_op_t **) a_voidp;
141 const remote_fencing_op_t **b = (const remote_fencing_op_t **) b_voidp;
142 gboolean a_pending = ((*a)->state != st_failed) && ((*a)->state != st_done);
143 gboolean b_pending = ((*b)->state != st_failed) && ((*b)->state != st_done);
144
145 if (a_pending && b_pending) {
146 return 0;
147 } else if (a_pending) {
148 return -1;
149 } else if (b_pending) {
150 return 1;
151 } else if ((*b)->completed == (*a)->completed) {
152 if ((*b)->completed_nsec > (*a)->completed_nsec) {
153 return 1;
154 } else if ((*b)->completed_nsec == (*a)->completed_nsec) {
155 return 0;
156 }
157 } else if ((*b)->completed > (*a)->completed) {
158 return 1;
159 }
160
161 return -1;
162 }
163
164
165
166
167
168
169
170 void
171 stonith_fence_history_trim(void)
172 {
173 guint num_ops;
174
175 if (!stonith_remote_op_list) {
176 return;
177 }
178 num_ops = g_hash_table_size(stonith_remote_op_list);
179 if (num_ops > MAX_STONITH_HISTORY) {
180 remote_fencing_op_t *ops[num_ops];
181 remote_fencing_op_t *op = NULL;
182 GHashTableIter iter;
183 int i;
184
185 crm_trace("Fencing History growing beyond limit of %d so purge "
186 "half of failed/successful attempts", MAX_STONITH_HISTORY);
187
188
189 i = 0;
190 g_hash_table_iter_init(&iter, stonith_remote_op_list);
191 while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
192 ops[i++] = op;
193 }
194
195
196
197 qsort(ops, num_ops, sizeof(remote_fencing_op_t *), op_time_sort);
198
199 for (i = MAX_STONITH_HISTORY / 2; i < num_ops; i++) {
200
201
202
203 if ((ops[i]->state == st_failed) || (ops[i]->state == st_done)) {
204 g_hash_table_remove(stonith_remote_op_list, ops[i]->id);
205 }
206 }
207
208
209
210 }
211 }
212
213
214
215
216
217
218
219
220
221 static GHashTable *
222 stonith_xml_history_to_list(xmlNode *history)
223 {
224 xmlNode *xml_op = NULL;
225 GHashTable *rv = NULL;
226
227 init_stonith_remote_op_hash_table(&rv);
228
229 CRM_LOG_ASSERT(rv != NULL);
230
231 for (xml_op = pcmk__xml_first_child(history); xml_op != NULL;
232 xml_op = pcmk__xml_next(xml_op)) {
233 remote_fencing_op_t *op = NULL;
234 char *id = crm_element_value_copy(xml_op, F_STONITH_REMOTE_OP_ID);
235 int state;
236 long long completed;
237 long long completed_nsec = 0L;
238
239 if (!id) {
240 crm_warn("Malformed fencing history received from peer");
241 continue;
242 }
243
244 crm_trace("Attaching op %s to hashtable", id);
245
246 op = calloc(1, sizeof(remote_fencing_op_t));
247
248 op->id = id;
249 op->target = crm_element_value_copy(xml_op, F_STONITH_TARGET);
250 op->action = crm_element_value_copy(xml_op, F_STONITH_ACTION);
251 op->originator = crm_element_value_copy(xml_op, F_STONITH_ORIGIN);
252 op->delegate = crm_element_value_copy(xml_op, F_STONITH_DELEGATE);
253 op->client_name = crm_element_value_copy(xml_op, F_STONITH_CLIENTNAME);
254 crm_element_value_ll(xml_op, F_STONITH_DATE, &completed);
255 op->completed = (time_t) completed;
256 crm_element_value_ll(xml_op, F_STONITH_DATE_NSEC, &completed_nsec);
257 op->completed_nsec = completed_nsec;
258 crm_element_value_int(xml_op, F_STONITH_STATE, &state);
259 op->state = (enum op_state) state;
260 stonith__xe_get_result(xml_op, &op->result);
261
262 g_hash_table_replace(rv, id, op);
263 CRM_LOG_ASSERT(g_hash_table_lookup(rv, id) != NULL);
264 }
265
266 return rv;
267 }
268
269
270
271
272
273
274
275
276
277
278
279
280
281 static xmlNode *
282 stonith_local_history_diff_and_merge(GHashTable *remote_history,
283 gboolean add_id,
284 const char *target)
285 {
286 xmlNode *history = NULL;
287 GHashTableIter iter;
288 remote_fencing_op_t *op = NULL;
289 gboolean updated = FALSE;
290 int cnt = 0;
291
292 if (stonith_remote_op_list) {
293 char *id = NULL;
294
295 history = create_xml_node(NULL, F_STONITH_HISTORY_LIST);
296
297 g_hash_table_iter_init(&iter, stonith_remote_op_list);
298 while (g_hash_table_iter_next(&iter, (void **)&id, (void **)&op)) {
299 xmlNode *entry = NULL;
300
301 if (remote_history) {
302 remote_fencing_op_t *remote_op =
303 g_hash_table_lookup(remote_history, op->id);
304
305 if (remote_op) {
306 if (stonith__op_state_pending(op->state)
307 && !stonith__op_state_pending(remote_op->state)) {
308
309 crm_debug("Updating outdated pending operation %.8s "
310 "(state=%s) according to the one (state=%s) from "
311 "remote peer history",
312 op->id, stonith_op_state_str(op->state),
313 stonith_op_state_str(remote_op->state));
314
315 g_hash_table_steal(remote_history, op->id);
316 op->id = remote_op->id;
317 remote_op->id = id;
318 g_hash_table_iter_replace(&iter, remote_op);
319
320 updated = TRUE;
321 continue;
322
323 } else if (!stonith__op_state_pending(op->state)
324 && stonith__op_state_pending(remote_op->state)) {
325
326 crm_debug("Broadcasting operation %.8s (state=%s) to "
327 "update the outdated pending one "
328 "(state=%s) in remote peer history",
329 op->id, stonith_op_state_str(op->state),
330 stonith_op_state_str(remote_op->state));
331
332 g_hash_table_remove(remote_history, op->id);
333
334 } else {
335 g_hash_table_remove(remote_history, op->id);
336 continue;
337 }
338 }
339 }
340
341 if (!pcmk__str_eq(target, op->target, pcmk__str_null_matches)) {
342 continue;
343 }
344
345 cnt++;
346 crm_trace("Attaching op %s", op->id);
347 entry = create_xml_node(history, STONITH_OP_EXEC);
348 if (add_id) {
349 crm_xml_add(entry, F_STONITH_REMOTE_OP_ID, op->id);
350 }
351 crm_xml_add(entry, F_STONITH_TARGET, op->target);
352 crm_xml_add(entry, F_STONITH_ACTION, op->action);
353 crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
354 crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
355 crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
356 crm_xml_add_ll(entry, F_STONITH_DATE, op->completed);
357 crm_xml_add_ll(entry, F_STONITH_DATE_NSEC, op->completed_nsec);
358 crm_xml_add_int(entry, F_STONITH_STATE, op->state);
359 stonith__xe_set_result(entry, &op->result);
360 }
361 }
362
363 if (remote_history) {
364 init_stonith_remote_op_hash_table(&stonith_remote_op_list);
365
366 updated |= g_hash_table_size(remote_history);
367
368 g_hash_table_iter_init(&iter, remote_history);
369 while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
370 if (stonith__op_state_pending(op->state) &&
371 pcmk__str_eq(op->originator, stonith_our_uname, pcmk__str_casei)) {
372
373 crm_warn("Failing pending operation %.8s originated by us but "
374 "known only from peer history", op->id);
375 op->state = st_failed;
376 set_fencing_completed(op);
377
378
379
380
381 pcmk__set_result(&op->result, CRM_EX_EXPIRED, PCMK_EXEC_INVALID,
382 "Initiated by earlier fencer "
383 "process and presumed failed");
384 fenced_broadcast_op_result(op, false);
385 }
386
387 g_hash_table_iter_steal(&iter);
388 g_hash_table_replace(stonith_remote_op_list, op->id, op);
389
390
391
392
393
394
395
396
397 }
398
399 g_hash_table_destroy(remote_history);
400 }
401
402 if (updated) {
403 stonith_fence_history_trim();
404 fenced_send_notification(T_STONITH_NOTIFY_HISTORY, NULL, NULL);
405 }
406
407 if (cnt == 0) {
408 free_xml(history);
409 return NULL;
410 } else {
411 return history;
412 }
413 }
414
415
416
417
418
419
420
421
422
423
424
425 static xmlNode *
426 stonith_local_history(gboolean add_id, const char *target)
427 {
428 return stonith_local_history_diff_and_merge(NULL, add_id, target);
429 }
430
431
432
433
434
435
436
437
438
439
440
441
442 void
443 stonith_fence_history(xmlNode *msg, xmlNode **output,
444 const char *remote_peer, int options)
445 {
446 const char *target = NULL;
447 xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_NEVER);
448 xmlNode *out_history = NULL;
449
450 if (dev) {
451 target = crm_element_value(dev, F_STONITH_TARGET);
452 if (target && (options & st_opt_cs_nodeid)) {
453 int nodeid;
454 crm_node_t *node;
455
456 pcmk__scan_min_int(target, &nodeid, 0);
457 node = pcmk__search_known_node_cache(nodeid, NULL, CRM_GET_PEER_ANY);
458 if (node) {
459 target = node->uname;
460 }
461 }
462 }
463
464 if (options & st_opt_cleanup) {
465 crm_trace("Cleaning up operations on %s in %p", target,
466 stonith_remote_op_list);
467
468 stonith_fence_history_cleanup(target,
469 crm_element_value(msg, F_STONITH_CALLID) != NULL);
470 } else if (options & st_opt_broadcast) {
471
472
473
474
475 fenced_send_notification(T_STONITH_NOTIFY_HISTORY_SYNCED, NULL, NULL);
476 if (crm_element_value(msg, F_STONITH_CALLID)) {
477
478
479
480
481
482
483 out_history = stonith_local_history(TRUE, NULL);
484 crm_trace("Broadcasting history to peers");
485 stonith_send_broadcast_history(out_history,
486 st_opt_broadcast | st_opt_discard_reply,
487 NULL);
488 } else if (remote_peer &&
489 !pcmk__str_eq(remote_peer, stonith_our_uname, pcmk__str_casei)) {
490 xmlNode *history = get_xpath_object("//" F_STONITH_HISTORY_LIST,
491 msg, LOG_NEVER);
492
493
494
495
496
497
498
499
500
501
502 if (!history || !pcmk__xe_attr_is_true(history, F_STONITH_DIFFERENTIAL)) {
503 GHashTable *received_history = NULL;
504
505 if (history != NULL) {
506 received_history = stonith_xml_history_to_list(history);
507 }
508 out_history =
509 stonith_local_history_diff_and_merge(received_history, TRUE, NULL);
510 if (out_history) {
511 crm_trace("Broadcasting history-diff to peers");
512 pcmk__xe_set_bool_attr(out_history, F_STONITH_DIFFERENTIAL, true);
513 stonith_send_broadcast_history(out_history,
514 st_opt_broadcast | st_opt_discard_reply,
515 NULL);
516 } else {
517 crm_trace("History-diff is empty - skip broadcast");
518 }
519 }
520 } else {
521 crm_trace("Skipping history-query-broadcast (%s%s)"
522 " we sent ourselves",
523 remote_peer?"remote-peer=":"local-ipc",
524 remote_peer?remote_peer:"");
525 }
526 } else {
527
528 crm_trace("Looking for operations on %s in %p", target,
529 stonith_remote_op_list);
530 *output = stonith_local_history(FALSE, target);
531 }
532 free_xml(out_history);
533 }