root/pengine/native.c

/* */

DEFINITIONS

1. native_choose_node
2. node_list_attr_score
3. node_hash_update
4. node_hash_dup
5. native_merge_weights
6. rsc_merge_weights
7. native_color
8. is_op_dup
9. RecurringOp
10. Recurring
11. RecurringOp_Stopped
12. Recurring_Stopped
13. handle_migration_actions
14. native_create_actions
15. rsc_avoids_remote_nodes
16. native_internal_constraints
17. native_rsc_colocation_lh
18. filter_colocation_constraint
19. influence_priority
20. colocation_match
21. native_rsc_colocation_rh
22. filter_rsc_ticket
23. rsc_ticket_constraint
24. native_action_flags
25. native_update_actions
26. native_rsc_location
27. native_expand
28. LogAction
29. LogActions
30. StopRsc
31. StartRsc
32. PromoteRsc
33. DemoteRsc
34. RoleError
35. NullOp
36. DeleteRsc
37. increment_clone
38. probe_anon_group_member
39. native_create_probe
40. native_start_constraints
41. native_stop_constraints
42. rsc_stonith_ordering
43. ReloadRsc
44. native_append_meta

   1 /*
   2  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  *
   9  * This software is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 
  19 #include <crm_internal.h>
  20 
  21 #include <pengine.h>
  22 #include <crm/pengine/rules.h>
  23 #include <crm/msg_xml.h>
  24 #include <allocate.h>
  25 #include <notif.h>
  26 #include <utils.h>
  27 #include <crm/services.h>
  28 
  29 /* #define DELETE_THEN_REFRESH 1  // The crmd will remove the resource from the CIB itself, making this redundant */
  30 #define INFINITY_HACK   (INFINITY * -100)
  31 
  32 #define VARIANT_NATIVE 1
  33 #include <lib/pengine/variant.h>
  34 
  35 gboolean update_action(action_t * then);
  36 void native_rsc_colocation_rh_must(resource_t * rsc_lh, gboolean update_lh,
  37                                    resource_t * rsc_rh, gboolean update_rh);
  38 
  39 void native_rsc_colocation_rh_mustnot(resource_t * rsc_lh, gboolean update_lh,
  40                                       resource_t * rsc_rh, gboolean update_rh);
  41 
  42 void Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set);
  43 void RecurringOp(resource_t * rsc, action_t * start, node_t * node,
  44                  xmlNode * operation, pe_working_set_t * data_set);
  45 void Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node,
  46                        pe_working_set_t * data_set);
  47 void RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node,
  48                          xmlNode * operation, pe_working_set_t * data_set);
  49 
  50 void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set);
  51 gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set);
  52 gboolean StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
  53 gboolean StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
  54 gboolean DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
  55 gboolean PromoteRsc(resource_t * rsc, node_t * next, gboolean optional,
  56                     pe_working_set_t * data_set);
  57 gboolean RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
  58 gboolean NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set);
  59 
  60 /* *INDENT-OFF* */
  61 enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
  62 /* Current State */
  63 /*       Next State:    Unknown           Stopped            Started            Slave             Master */
  64     /* Unknown */ { RSC_ROLE_UNKNOWN, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, },
  65     /* Stopped */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE,   RSC_ROLE_SLAVE, },
  66     /* Started */ { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STARTED, RSC_ROLE_SLAVE,   RSC_ROLE_MASTER, },
  67     /* Slave */   { RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_STOPPED, RSC_ROLE_SLAVE,   RSC_ROLE_MASTER, },
  68     /* Master */  { RSC_ROLE_STOPPED, RSC_ROLE_SLAVE,   RSC_ROLE_SLAVE,   RSC_ROLE_SLAVE,   RSC_ROLE_MASTER, },
  69 };
  70 
  71 gboolean (*rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX])(resource_t*,node_t*,gboolean,pe_working_set_t*) = {
  72 /* Current State */
  73 /*       Next State:       Unknown      Stopped         Started         Slave           Master */
  74     /* Unknown */       { RoleError,    StopRsc,        RoleError,      RoleError,      RoleError,  },
  75     /* Stopped */       { RoleError,    NullOp,         StartRsc,       StartRsc,       RoleError,  },
  76     /* Started */       { RoleError,    StopRsc,        NullOp,         NullOp,         PromoteRsc, },
  77     /* Slave */         { RoleError,    StopRsc,        StopRsc,        NullOp,         PromoteRsc, },
  78     /* Master */        { RoleError,    DemoteRsc,      DemoteRsc,      DemoteRsc,      NullOp,     },
  79 };
  80 /* *INDENT-ON* */
  81 
  82 static gboolean
  83 native_choose_node(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
     /*  */
  84 {
  85     GListPtr nodes = NULL;
  86     node_t *chosen = NULL;
  87     node_t *best = NULL;
  88     int multiple = 1;
  89     int length = 0;
  90     gboolean result = FALSE;
  91 
  92     process_utilization(rsc, &prefer, data_set);
  93 
  94     if (is_not_set(rsc->flags, pe_rsc_provisional)) {
  95         return rsc->allocated_to ? TRUE : FALSE;
  96     }
  97 
  98     // Sort allowed nodes by weight
  99     if (rsc->allowed_nodes) {
 100         length = g_hash_table_size(rsc->allowed_nodes);
 101     }
 102     if (length > 0) {
 103         nodes = g_hash_table_get_values(rsc->allowed_nodes);
 104         nodes = g_list_sort_with_data(nodes, sort_node_weight,
 105                                       g_list_nth_data(rsc->running_on, 0));
 106 
 107         // First node in sorted list has the best score
 108         best = g_list_nth_data(nodes, 0);
 109     }
 110 
 111     if (prefer && nodes) {
 112         chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
 113 
 114         if (chosen == NULL) {
 115             pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
 116                          prefer->details->uname, rsc->id);
 117 
 118         /* Favor the preferred node as long as its weight is at least as good as
 119          * the best allowed node's.
 120          *
 121          * An alternative would be to favor the preferred node even if the best
 122          * node is better, when the best node's weight is less than INFINITY.
 123          */
 124         } else if ((chosen->weight < 0) || (chosen->weight < best->weight)) {
 125             pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
 126                          chosen->details->uname, rsc->id);
 127             chosen = NULL;
 128 
 129         } else if (!can_run_resources(chosen)) {
 130             pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
 131                          chosen->details->uname, rsc->id);
 132             chosen = NULL;
 133 
 134         } else {
 135             pe_rsc_trace(rsc,
 136                          "Chose preferred node %s for %s (ignoring %d candidates)",
 137                          chosen->details->uname, rsc->id, length);
 138         }
 139     }
 140 
 141     if ((chosen == NULL) && nodes) {
 142         /* Either there is no preferred node, or the preferred node is not
 143          * available, but there are other nodes allowed to run the resource.
 144          */
 145 
 146         chosen = best;
 147         pe_rsc_trace(rsc, "Chose node %s for %s from %d candidates",
 148                      chosen ? chosen->details->uname : "<none>", rsc->id, length);
 149 
 150         if (!pe_rsc_is_unique_clone(rsc->parent)
 151             && chosen && (chosen->weight > 0) && can_run_resources(chosen)) {
 152             /* If the resource is already running on a node, prefer that node if
 153              * it is just as good as the chosen node.
 154              *
 155              * We don't do this for unique clone instances, because
 156              * distribute_children() has already assigned instances to their
 157              * running nodes when appropriate, and if we get here, we don't want
 158              * remaining unallocated instances to prefer a node that's already
 159              * running another instance.
 160              */
 161             node_t *running = g_list_nth_data(rsc->running_on, 0);
 162 
 163             if (running && (can_run_resources(running) == FALSE)) {
 164                 pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
 165                              rsc->id, running->details->uname);
 166             } else if (running) {
 167                 for (GList *iter = nodes->next; iter; iter = iter->next) {
 168                     node_t *tmp = (node_t *) iter->data;
 169 
 170                     if (tmp->weight != chosen->weight) {
 171                         // The nodes are sorted by weight, so no more are equal
 172                         break;
 173                     }
 174                     if (tmp->details == running->details) {
 175                         // Scores are equal, so prefer the current node
 176                         chosen = tmp;
 177                     }
 178                     multiple++;
 179                 }
 180             }
 181         }
 182     }
 183 
 184     if (multiple > 1) {
 185         static char score[33];
 186         int log_level = (chosen->weight >= INFINITY)? LOG_WARNING : LOG_INFO;
 187 
 188         score2char_stack(chosen->weight, score, sizeof(score));
 189         do_crm_log(log_level,
 190                    "Chose node %s for %s from %d nodes with score %s",
 191                    chosen->details->uname, rsc->id, multiple, score);
 192     }
 193 
 194     result = native_assign_node(rsc, nodes, chosen, FALSE);
 195     g_list_free(nodes);
 196     return result;
 197 }
 198 
 199 static int
 200 node_list_attr_score(GHashTable * list, const char *attr, const char *value)
     /*  */
 201 {
 202     GHashTableIter iter;
 203     node_t *node = NULL;
 204     int best_score = -INFINITY;
 205     const char *best_node = NULL;
 206 
 207     if (attr == NULL) {
 208         attr = CRM_ATTR_UNAME;
 209     }
 210 
 211     g_hash_table_iter_init(&iter, list);
 212     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
 213         int weight = node->weight;
 214 
 215         if (can_run_resources(node) == FALSE) {
 216             weight = -INFINITY;
 217         }
 218         if (weight > best_score || best_node == NULL) {
 219             const char *tmp = pe_node_attribute_raw(node, attr);
 220 
 221             if (safe_str_eq(value, tmp)) {
 222                 best_score = weight;
 223                 best_node = node->details->uname;
 224             }
 225         }
 226     }
 227 
 228     if (safe_str_neq(attr, CRM_ATTR_UNAME)) {
 229         crm_info("Best score for %s=%s was %s with %d",
 230                  attr, value, best_node ? best_node : "<none>", best_score);
 231     }
 232 
 233     return best_score;
 234 }
 235 
 236 static void
 237 node_hash_update(GHashTable * list1, GHashTable * list2, const char *attr, float factor,
     /*  */
 238                  gboolean only_positive)
 239 {
 240     int score = 0;
 241     int new_score = 0;
 242     GHashTableIter iter;
 243     node_t *node = NULL;
 244 
 245     if (attr == NULL) {
 246         attr = CRM_ATTR_UNAME;
 247     }
 248 
 249     g_hash_table_iter_init(&iter, list1);
 250     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
 251         float weight_f = 0;
 252         int weight = 0;
 253 
 254         CRM_LOG_ASSERT(node != NULL);
 255         if(node == NULL) { continue; };
 256 
 257         score = node_list_attr_score(list2, attr, pe_node_attribute_raw(node, attr));
 258 
 259         weight_f = factor * score;
 260         /* Round the number */
 261         /* http://c-faq.com/fp/round.html */
 262         weight = (int)(weight_f < 0 ? weight_f - 0.5 : weight_f + 0.5);
 263 
 264         new_score = merge_weights(weight, node->weight);
 265 
 266         if (factor < 0 && score < 0) {
 267             /* Negative preference for a node with a negative score
 268              * should not become a positive preference
 269              *
 270              * TODO - Decide if we want to filter only if weight == -INFINITY
 271              *
 272              */
 273             crm_trace("%s: Filtering %d + %f*%d (factor * score)",
 274                       node->details->uname, node->weight, factor, score);
 275 
 276         } else if (node->weight == INFINITY_HACK) {
 277             crm_trace("%s: Filtering %d + %f*%d (node < 0)",
 278                       node->details->uname, node->weight, factor, score);
 279 
 280         } else if (only_positive && new_score < 0 && node->weight > 0) {
 281             node->weight = INFINITY_HACK;
 282             crm_trace("%s: Filtering %d + %f*%d (score > 0)",
 283                       node->details->uname, node->weight, factor, score);
 284 
 285         } else if (only_positive && new_score < 0 && node->weight == 0) {
 286             crm_trace("%s: Filtering %d + %f*%d (score == 0)",
 287                       node->details->uname, node->weight, factor, score);
 288 
 289         } else {
 290             crm_trace("%s: %d + %f*%d", node->details->uname, node->weight, factor, score);
 291             node->weight = new_score;
 292         }
 293     }
 294 }
 295 
 296 GHashTable *
 297 node_hash_dup(GHashTable * hash)
     /*  */
 298 {
 299     /* Hack! */
 300     GListPtr list = g_hash_table_get_values(hash);
 301     GHashTable *result = node_hash_from_list(list);
 302 
 303     g_list_free(list);
 304     return result;
 305 }
 306 
 307 GHashTable *
 308 native_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, const char *attr,
     /*  */
 309                      float factor, enum pe_weights flags)
 310 {
 311     return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags);
 312 }
 313 
 314 GHashTable *
 315 rsc_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, const char *attr,
     /*  */
 316                   float factor, enum pe_weights flags)
 317 {
 318     GHashTable *work = NULL;
 319     int multiplier = 1;
 320 
 321     if (factor < 0) {
 322         multiplier = -1;
 323     }
 324 
 325     if (is_set(rsc->flags, pe_rsc_merging)) {
 326         pe_rsc_info(rsc, "%s: Breaking dependency loop at %s", rhs, rsc->id);
 327         return nodes;
 328     }
 329 
 330     set_bit(rsc->flags, pe_rsc_merging);
 331 
 332     if (is_set(flags, pe_weights_init)) {
 333         if (rsc->variant == pe_group && rsc->children) {
 334             GListPtr last = rsc->children;
 335 
 336             while (last->next != NULL) {
 337                 last = last->next;
 338             }
 339 
 340             pe_rsc_trace(rsc, "Merging %s as a group %p %p", rsc->id, rsc->children, last);
 341             work = rsc_merge_weights(last->data, rhs, NULL, attr, factor, flags);
 342 
 343         } else {
 344             work = node_hash_dup(rsc->allowed_nodes);
 345         }
 346         clear_bit(flags, pe_weights_init);
 347 
 348     } else if (rsc->variant == pe_group && rsc->children) {
 349         GListPtr iter = rsc->children;
 350 
 351         pe_rsc_trace(rsc, "%s: Combining scores from %d children of %s", rhs, g_list_length(iter), rsc->id);
 352         work = node_hash_dup(nodes);
 353         for(iter = rsc->children; iter->next != NULL; iter = iter->next) {
 354             work = rsc_merge_weights(iter->data, rhs, work, attr, factor, flags);
 355         }
 356 
 357     } else {
 358         pe_rsc_trace(rsc, "%s: Combining scores from %s", rhs, rsc->id);
 359         work = node_hash_dup(nodes);
 360         node_hash_update(work, rsc->allowed_nodes, attr, factor,
 361                          is_set(flags, pe_weights_positive));
 362     }
 363 
 364     if (is_set(flags, pe_weights_rollback) && can_run_any(work) == FALSE) {
 365         pe_rsc_info(rsc, "%s: Rolling back scores from %s", rhs, rsc->id);
 366         g_hash_table_destroy(work);
 367         clear_bit(rsc->flags, pe_rsc_merging);
 368         return nodes;
 369     }
 370 
 371     if (can_run_any(work)) {
 372         GListPtr gIter = NULL;
 373 
 374         if (is_set(flags, pe_weights_forward)) {
 375             gIter = rsc->rsc_cons;
 376             crm_trace("Checking %d additional colocation constraints", g_list_length(gIter));
 377 
 378         } else if(rsc->variant == pe_group && rsc->children) {
 379             GListPtr last = rsc->children;
 380 
 381             while (last->next != NULL) {
 382                 last = last->next;
 383             }
 384 
 385             gIter = ((resource_t*)last->data)->rsc_cons_lhs;
 386             crm_trace("Checking %d additional optional group colocation constraints from %s",
 387                       g_list_length(gIter), ((resource_t*)last->data)->id);
 388 
 389         } else {
 390             gIter = rsc->rsc_cons_lhs;
 391             crm_trace("Checking %d additional optional colocation constraints %s", g_list_length(gIter), rsc->id);
 392         }
 393 
 394         for (; gIter != NULL; gIter = gIter->next) {
 395             resource_t *other = NULL;
 396             rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
 397 
 398             if (is_set(flags, pe_weights_forward)) {
 399                 other = constraint->rsc_rh;
 400             } else {
 401                 other = constraint->rsc_lh;
 402             }
 403 
 404             pe_rsc_trace(rsc, "Applying %s (%s)", constraint->id, other->id);
 405             work = rsc_merge_weights(other, rhs, work, constraint->node_attribute,
 406                                      multiplier * (float)constraint->score / INFINITY, flags|pe_weights_rollback);
 407             dump_node_scores(LOG_TRACE, NULL, rhs, work);
 408         }
 409 
 410     }
 411 
 412     if (is_set(flags, pe_weights_positive)) {
 413         node_t *node = NULL;
 414         GHashTableIter iter;
 415 
 416         g_hash_table_iter_init(&iter, work);
 417         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
 418             if (node->weight == INFINITY_HACK) {
 419                 node->weight = 1;
 420             }
 421         }
 422     }
 423 
 424     if (nodes) {
 425         g_hash_table_destroy(nodes);
 426     }
 427 
 428     clear_bit(rsc->flags, pe_rsc_merging);
 429     return work;
 430 }
 431 
 432 node_t *
 433 native_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
     /*  */
 434 {
 435     GListPtr gIter = NULL;
 436     int alloc_details = scores_log_level + 1;
 437 
 438     if (rsc->parent && is_not_set(rsc->parent->flags, pe_rsc_allocating)) {
 439         /* never allocate children on their own */
 440         pe_rsc_debug(rsc, "Escalating allocation of %s to its parent: %s", rsc->id,
 441                      rsc->parent->id);
 442         rsc->parent->cmds->allocate(rsc->parent, prefer, data_set);
 443     }
 444 
 445     if (is_not_set(rsc->flags, pe_rsc_provisional)) {
 446         return rsc->allocated_to;
 447     }
 448 
 449     if (is_set(rsc->flags, pe_rsc_allocating)) {
 450         pe_rsc_debug(rsc, "Dependency loop detected involving %s", rsc->id);
 451         return NULL;
 452     }
 453 
 454     set_bit(rsc->flags, pe_rsc_allocating);
 455     print_resource(alloc_details, "Allocating: ", rsc, FALSE);
 456     dump_node_scores(alloc_details, rsc, "Pre-alloc", rsc->allowed_nodes);
 457 
 458     for (gIter = rsc->rsc_cons; gIter != NULL; gIter = gIter->next) {
 459         rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
 460 
 461         GHashTable *archive = NULL;
 462         resource_t *rsc_rh = constraint->rsc_rh;
 463 
 464         pe_rsc_trace(rsc, "%s: Pre-Processing %s (%s, %d, %s)",
 465                      rsc->id, constraint->id, rsc_rh->id,
 466                      constraint->score, role2text(constraint->role_lh));
 467         if (constraint->role_lh >= RSC_ROLE_MASTER
 468             || (constraint->score < 0 && constraint->score > -INFINITY)) {
 469             archive = node_hash_dup(rsc->allowed_nodes);
 470         }
 471         rsc_rh->cmds->allocate(rsc_rh, NULL, data_set);
 472         rsc->cmds->rsc_colocation_lh(rsc, rsc_rh, constraint);
 473         if (archive && can_run_any(rsc->allowed_nodes) == FALSE) {
 474             pe_rsc_info(rsc, "%s: Rolling back scores from %s", rsc->id, rsc_rh->id);
 475             g_hash_table_destroy(rsc->allowed_nodes);
 476             rsc->allowed_nodes = archive;
 477             archive = NULL;
 478         }
 479         if (archive) {
 480             g_hash_table_destroy(archive);
 481         }
 482     }
 483 
 484     dump_node_scores(alloc_details, rsc, "Post-coloc", rsc->allowed_nodes);
 485 
 486     for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) {
 487         rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
 488 
 489         rsc->allowed_nodes =
 490             constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id, rsc->allowed_nodes,
 491                                                     constraint->node_attribute,
 492                                                     (float)constraint->score / INFINITY,
 493                                                     pe_weights_rollback);
 494     }
 495 
 496     print_resource(LOG_DEBUG_2, "Allocating: ", rsc, FALSE);
 497     if (rsc->next_role == RSC_ROLE_STOPPED) {
 498         pe_rsc_trace(rsc, "Making sure %s doesn't get allocated", rsc->id);
 499         /* make sure it doesn't come up again */
 500         resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE, data_set);
 501 
 502     } else if(rsc->next_role > rsc->role
 503               && is_set(data_set->flags, pe_flag_have_quorum) == FALSE
 504               && data_set->no_quorum_policy == no_quorum_freeze) {
 505         crm_notice("Resource %s cannot be elevated from %s to %s: no-quorum-policy=freeze",
 506                    rsc->id, role2text(rsc->role), role2text(rsc->next_role));
 507         rsc->next_role = rsc->role;
 508     }
 509 
 510     dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__,
 511                      rsc->allowed_nodes);
 512     if (is_set(data_set->flags, pe_flag_stonith_enabled)
 513         && is_set(data_set->flags, pe_flag_have_stonith_resource) == FALSE) {
 514         clear_bit(rsc->flags, pe_rsc_managed);
 515     }
 516 
 517     if (is_not_set(rsc->flags, pe_rsc_managed)) {
 518         const char *reason = NULL;
 519         node_t *assign_to = NULL;
 520 
 521         rsc->next_role = rsc->role;
 522         if (rsc->running_on == NULL) {
 523             reason = "inactive";
 524         } else if (rsc->role == RSC_ROLE_MASTER) {
 525             assign_to = rsc->running_on->data;
 526             reason = "master";
 527         } else if (is_set(rsc->flags, pe_rsc_failed)) {
 528             assign_to = rsc->running_on->data;
 529             reason = "failed";
 530         } else {
 531             assign_to = rsc->running_on->data;
 532             reason = "active";
 533         }
 534         pe_rsc_info(rsc, "Unmanaged resource %s allocated to %s: %s", rsc->id,
 535                     (assign_to? assign_to->details->uname : "no node"), reason);
 536         native_assign_node(rsc, NULL, assign_to, TRUE);
 537 
 538     } else if (is_set(data_set->flags, pe_flag_stop_everything)) {
 539         pe_rsc_debug(rsc, "Forcing %s to stop", rsc->id);
 540         native_assign_node(rsc, NULL, NULL, TRUE);
 541 
 542     } else if (is_set(rsc->flags, pe_rsc_provisional)
 543                && native_choose_node(rsc, prefer, data_set)) {
 544         pe_rsc_trace(rsc, "Allocated resource %s to %s", rsc->id,
 545                      rsc->allocated_to->details->uname);
 546 
 547     } else if (rsc->allocated_to == NULL) {
 548         if (is_not_set(rsc->flags, pe_rsc_orphan)) {
 549             pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
 550         } else if (rsc->running_on != NULL) {
 551             pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
 552         }
 553 
 554     } else {
 555         pe_rsc_debug(rsc, "Pre-Allocated resource %s to %s", rsc->id,
 556                      rsc->allocated_to->details->uname);
 557     }
 558 
 559     clear_bit(rsc->flags, pe_rsc_allocating);
 560     print_resource(LOG_DEBUG_3, "Allocated ", rsc, TRUE);
 561 
 562     if (rsc->is_remote_node) {
 563         node_t *remote_node = pe_find_node(data_set->nodes, rsc->id);
 564 
 565         CRM_ASSERT(remote_node != NULL);
 566         if (rsc->allocated_to && rsc->next_role != RSC_ROLE_STOPPED) {
 567             crm_trace("Setting Pacemaker Remote node %s to ONLINE",
 568                       remote_node->details->id);
 569             remote_node->details->online = TRUE;
 570             /* We shouldn't consider an unseen remote-node unclean if we are going
 571              * to try and connect to it. Otherwise we get an unnecessary fence */
 572             if (remote_node->details->unseen == TRUE) {
 573                 remote_node->details->unclean = FALSE;
 574             }
 575 
 576         } else {
 577             crm_trace("Setting Pacemaker Remote node %s to SHUTDOWN (next role %s, %sallocated)",
 578                       remote_node->details->id, role2text(rsc->next_role),
 579                       (rsc->allocated_to? "" : "un"));
 580             remote_node->details->shutdown = TRUE;
 581         }
 582     }
 583 
 584     return rsc->allocated_to;
 585 }
 586 
 587 static gboolean
 588 is_op_dup(resource_t * rsc, const char *name, const char *interval)
     /*  */
 589 {
 590     gboolean dup = FALSE;
 591     const char *id = NULL;
 592     const char *value = NULL;
 593     xmlNode *operation = NULL;
 594 
 595     CRM_ASSERT(rsc);
 596     for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
 597          operation = __xml_next_element(operation)) {
 598         if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
 599             value = crm_element_value(operation, "name");
 600             if (safe_str_neq(value, name)) {
 601                 continue;
 602             }
 603 
 604             value = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
 605             if (value == NULL) {
 606                 value = "0";
 607             }
 608 
 609             if (safe_str_neq(value, interval)) {
 610                 continue;
 611             }
 612 
 613             if (id == NULL) {
 614                 id = ID(operation);
 615 
 616             } else {
 617                 crm_config_err("Operation %s is a duplicate of %s", ID(operation), id);
 618                 crm_config_err
 619                     ("Do not use the same (name, interval) combination more than once per resource");
 620                 dup = TRUE;
 621             }
 622         }
 623     }
 624 
 625     return dup;
 626 }
 627 
 628 void
 629 RecurringOp(resource_t * rsc, action_t * start, node_t * node,
     /*  */
 630             xmlNode * operation, pe_working_set_t * data_set)
 631 {
 632     char *key = NULL;
 633     const char *name = NULL;
 634     const char *value = NULL;
 635     const char *interval = NULL;
 636     const char *node_uname = NULL;
 637 
 638     unsigned long long interval_ms = 0;
 639     action_t *mon = NULL;
 640     gboolean is_optional = TRUE;
 641     GListPtr possible_matches = NULL;
 642 
 643     /* Only process for the operations without role="Stopped" */
 644     value = crm_element_value(operation, "role");
 645     if (value && text2role(value) == RSC_ROLE_STOPPED) {
 646         return;
 647     }
 648 
 649     CRM_ASSERT(rsc);
 650     pe_rsc_trace(rsc, "Creating recurring action %s for %s in role %s on %s",
 651                  ID(operation), rsc->id, role2text(rsc->next_role),
 652                  node ? node->details->uname : "n/a");
 653 
 654     if (node != NULL) {
 655         node_uname = node->details->uname;
 656     }
 657 
 658     interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
 659     interval_ms = crm_get_interval(interval);
 660 
 661     if (interval_ms == 0) {
 662         return;
 663     }
 664 
 665     name = crm_element_value(operation, "name");
 666     if (is_op_dup(rsc, name, interval)) {
 667         return;
 668     }
 669 
 670     if (safe_str_eq(name, RSC_STOP)
 671         || safe_str_eq(name, RSC_START)
 672         || safe_str_eq(name, RSC_DEMOTE)
 673         || safe_str_eq(name, RSC_PROMOTE)
 674         ) {
 675         crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name);
 676         return;
 677     }
 678 
 679     key = generate_op_key(rsc->id, name, interval_ms);
 680     if (find_rsc_op_entry(rsc, key) == NULL) {
 681         /* disabled */
 682         free(key);
 683         return;
 684     }
 685 
 686     if (start != NULL) {
 687         pe_rsc_trace(rsc, "Marking %s %s due to %s",
 688                      key, is_set(start->flags, pe_action_optional) ? "optional" : "mandatory",
 689                      start->uuid);
 690         is_optional = (rsc->cmds->action_flags(start, NULL) & pe_action_optional);
 691     } else {
 692         pe_rsc_trace(rsc, "Marking %s optional", key);
 693         is_optional = TRUE;
 694     }
 695 
 696     /* start a monitor for an already active resource */
 697     possible_matches = find_actions_exact(rsc->actions, key, node);
 698     if (possible_matches == NULL) {
 699         is_optional = FALSE;
 700         pe_rsc_trace(rsc, "Marking %s mandatory: not active", key);
 701 
 702     } else {
 703         GListPtr gIter = NULL;
 704 
 705         for (gIter = possible_matches; gIter != NULL; gIter = gIter->next) {
 706             action_t *op = (action_t *) gIter->data;
 707 
 708             if (is_set(op->flags, pe_action_reschedule)) {
 709                 is_optional = FALSE;
 710                 break;
 711             }
 712         }
 713         g_list_free(possible_matches);
 714     }
 715 
 716     if ((rsc->next_role == RSC_ROLE_MASTER && value == NULL)
 717         || (value != NULL && text2role(value) != rsc->next_role)) {
 718         int log_level = LOG_DEBUG_2;
 719         const char *result = "Ignoring";
 720 
 721         if (is_optional) {
 722             char *local_key = strdup(key);
 723 
 724             log_level = LOG_INFO;
 725             result = "Cancelling";
 726             /* it's running : cancel it */
 727 
 728             mon = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set);
 729 
 730             free(mon->task);
 731             free(mon->cancel_task);
 732             mon->task = strdup(RSC_CANCEL);
 733             mon->cancel_task = strdup(name);
 734             add_hash_param(mon->meta, XML_LRM_ATTR_INTERVAL, interval);
 735             add_hash_param(mon->meta, XML_LRM_ATTR_TASK, name);
 736 
 737             local_key = NULL;
 738 
 739             switch (rsc->role) {
 740                 case RSC_ROLE_SLAVE:
 741                 case RSC_ROLE_STARTED:
 742                     if (rsc->next_role == RSC_ROLE_MASTER) {
 743                         local_key = promote_key(rsc);
 744 
 745                     } else if (rsc->next_role == RSC_ROLE_STOPPED) {
 746                         local_key = stop_key(rsc);
 747                     }
 748 
 749                     break;
 750                 case RSC_ROLE_MASTER:
 751                     local_key = demote_key(rsc);
 752                     break;
 753                 default:
 754                     break;
 755             }
 756 
 757             if (local_key) {
 758                 custom_action_order(rsc, NULL, mon, rsc, local_key, NULL,
 759                                     pe_order_runnable_left, data_set);
 760             }
 761 
 762             mon = NULL;
 763         }
 764 
 765         do_crm_log(log_level, "%s action %s (%s vs. %s)",
 766                    result, key, value ? value : role2text(RSC_ROLE_SLAVE),
 767                    role2text(rsc->next_role));
 768 
 769         free(key);
 770         return;
 771     }
 772 
 773     mon = custom_action(rsc, key, name, node, is_optional, TRUE, data_set);
 774     key = mon->uuid;
 775     if (is_optional) {
 776         pe_rsc_trace(rsc, "%s\t   %s (optional)", crm_str(node_uname), mon->uuid);
 777     }
 778 
 779     if (start == NULL || is_set(start->flags, pe_action_runnable) == FALSE) {
 780         pe_rsc_debug(rsc, "%s\t   %s (cancelled : start un-runnable)", crm_str(node_uname),
 781                      mon->uuid);
 782         update_action_flags(mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
 783 
 784     } else if (node == NULL || node->details->online == FALSE || node->details->unclean) {
 785         pe_rsc_debug(rsc, "%s\t   %s (cancelled : no node available)", crm_str(node_uname),
 786                      mon->uuid);
 787         update_action_flags(mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
 788 
 789     } else if (is_set(mon->flags, pe_action_optional) == FALSE) {
 790         pe_rsc_info(rsc, " Start recurring %s (%llus) for %s on %s", mon->task, interval_ms / 1000,
 791                     rsc->id, crm_str(node_uname));
 792     }
 793 
 794     if (rsc->next_role == RSC_ROLE_MASTER) {
 795         char *running_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);
 796 
 797         add_hash_param(mon->meta, XML_ATTR_TE_TARGET_RC, running_master);
 798         free(running_master);
 799     }
 800 
 801     if (node == NULL || is_set(rsc->flags, pe_rsc_managed)) {
 802         custom_action_order(rsc, start_key(rsc), NULL,
 803                             NULL, strdup(key), mon,
 804                             pe_order_implies_then | pe_order_runnable_left, data_set);
 805 
 806         custom_action_order(rsc, reload_key(rsc), NULL,
 807                             NULL, strdup(key), mon,
 808                             pe_order_implies_then | pe_order_runnable_left, data_set);
 809 
 810         if (rsc->next_role == RSC_ROLE_MASTER) {
 811             custom_action_order(rsc, promote_key(rsc), NULL,
 812                                 rsc, NULL, mon,
 813                                 pe_order_optional | pe_order_runnable_left, data_set);
 814 
 815         } else if (rsc->role == RSC_ROLE_MASTER) {
 816             custom_action_order(rsc, demote_key(rsc), NULL,
 817                                 rsc, NULL, mon,
 818                                 pe_order_optional | pe_order_runnable_left, data_set);
 819         }
 820     }
 821 }
 822 
 823 void
 824 Recurring(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set)
     /*  */
 825 {
 826     if (is_not_set(rsc->flags, pe_rsc_maintenance) &&
 827         (node == NULL || node->details->maintenance == FALSE)) {
 828         xmlNode *operation = NULL;
 829 
 830         for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
 831              operation = __xml_next_element(operation)) {
 832             if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
 833                 RecurringOp(rsc, start, node, operation, data_set);
 834             }
 835         }
 836     }
 837 }
 838 
 839 void
 840 RecurringOp_Stopped(resource_t * rsc, action_t * start, node_t * node,
     /*  */
 841                     xmlNode * operation, pe_working_set_t * data_set)
 842 {
 843     char *key = NULL;
 844     const char *name = NULL;
 845     const char *role = NULL;
 846     const char *interval = NULL;
 847     const char *node_uname = NULL;
 848 
 849     unsigned long long interval_ms = 0;
 850     GListPtr possible_matches = NULL;
 851     GListPtr gIter = NULL;
 852 
 853     /* TODO: Support of non-unique clone */
 854     if (is_set(rsc->flags, pe_rsc_unique) == FALSE) {
 855         return;
 856     }
 857 
 858     /* Only process for the operations with role="Stopped" */
 859     role = crm_element_value(operation, "role");
 860     if (role == NULL || text2role(role) != RSC_ROLE_STOPPED) {
 861         return;
 862     }
 863 
 864     pe_rsc_trace(rsc,
 865                  "Creating recurring actions %s for %s in role %s on nodes where it'll not be running",
 866                  ID(operation), rsc->id, role2text(rsc->next_role));
 867 
 868     if (node != NULL) {
 869         node_uname = node->details->uname;
 870     }
 871 
 872     interval = crm_element_value(operation, XML_LRM_ATTR_INTERVAL);
 873     interval_ms = crm_get_interval(interval);
 874 
 875     if (interval_ms == 0) {
 876         return;
 877     }
 878 
 879     name = crm_element_value(operation, "name");
 880     if (is_op_dup(rsc, name, interval)) {
 881         return;
 882     }
 883 
 884     if (safe_str_eq(name, RSC_STOP)
 885         || safe_str_eq(name, RSC_START)
 886         || safe_str_eq(name, RSC_DEMOTE)
 887         || safe_str_eq(name, RSC_PROMOTE)
 888         ) {
 889         crm_config_err("Invalid recurring action %s wth name: '%s'", ID(operation), name);
 890         return;
 891     }
 892 
 893     key = generate_op_key(rsc->id, name, interval_ms);
 894     if (find_rsc_op_entry(rsc, key) == NULL) {
 895         /* disabled */
 896         free(key);
 897         return;
 898     }
 899 
 900     /* if the monitor exists on the node where the resource will be running, cancel it */
 901     if (node != NULL) {
 902         possible_matches = find_actions_exact(rsc->actions, key, node);
 903         if (possible_matches) {
 904             action_t *cancel_op = NULL;
 905             char *local_key = strdup(key);
 906 
 907             g_list_free(possible_matches);
 908 
 909             cancel_op = custom_action(rsc, local_key, RSC_CANCEL, node, FALSE, TRUE, data_set);
 910 
 911             free(cancel_op->task);
 912             free(cancel_op->cancel_task);
 913             cancel_op->task = strdup(RSC_CANCEL);
 914             cancel_op->cancel_task = strdup(name);
 915             add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL, interval);
 916             add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, name);
 917 
 918             local_key = NULL;
 919 
 920             if (rsc->next_role == RSC_ROLE_STARTED || rsc->next_role == RSC_ROLE_SLAVE) {
 921                 /* rsc->role == RSC_ROLE_STOPPED: cancel the monitor before start */
 922                 /* rsc->role == RSC_ROLE_STARTED: for a migration, cancel the monitor on the target node before start */
 923                 custom_action_order(rsc, NULL, cancel_op, rsc, start_key(rsc), NULL,
 924                                     pe_order_runnable_left, data_set);
 925             }
 926 
 927             pe_rsc_info(rsc, "Cancel action %s (%s vs. %s) on %s",
 928                         key, role, role2text(rsc->next_role), crm_str(node_uname));
 929         }
 930     }
 931 
 932     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
 933         node_t *stop_node = (node_t *) gIter->data;
 934         const char *stop_node_uname = stop_node->details->uname;
 935         gboolean is_optional = TRUE;
 936         gboolean probe_is_optional = TRUE;
 937         gboolean stop_is_optional = TRUE;
 938         action_t *stopped_mon = NULL;
 939         char *rc_inactive = NULL;
 940         GListPtr probe_complete_ops = NULL;
 941         GListPtr stop_ops = NULL;
 942         GListPtr local_gIter = NULL;
 943         char *stop_op_key = NULL;
 944 
 945         if (node_uname && safe_str_eq(stop_node_uname, node_uname)) {
 946             continue;
 947         }
 948 
 949         pe_rsc_trace(rsc, "Creating recurring action %s for %s on %s",
 950                      ID(operation), rsc->id, crm_str(stop_node_uname));
 951 
 952         /* start a monitor for an already stopped resource */
 953         possible_matches = find_actions_exact(rsc->actions, key, stop_node);
 954         if (possible_matches == NULL) {
 955             pe_rsc_trace(rsc, "Marking %s mandatory on %s: not active", key,
 956                          crm_str(stop_node_uname));
 957             is_optional = FALSE;
 958         } else {
 959             pe_rsc_trace(rsc, "Marking %s optional on %s: already active", key,
 960                          crm_str(stop_node_uname));
 961             is_optional = TRUE;
 962             g_list_free(possible_matches);
 963         }
 964 
 965         stopped_mon = custom_action(rsc, strdup(key), name, stop_node, is_optional, TRUE, data_set);
 966 
 967         rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
 968         add_hash_param(stopped_mon->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
 969         free(rc_inactive);
 970 
 971         if (is_set(rsc->flags, pe_rsc_managed)) {
 972             char *probe_key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0);
 973             GListPtr probes = find_actions(rsc->actions, probe_key, stop_node);
 974             GListPtr pIter = NULL;
 975 
 976             for (pIter = probes; pIter != NULL; pIter = pIter->next) {
 977                 action_t *probe = (action_t *) pIter->data;
 978 
 979                 order_actions(probe, stopped_mon, pe_order_runnable_left);
 980                 crm_trace("%s then %s on %s", probe->uuid, stopped_mon->uuid, stop_node->details->uname);
 981             }
 982 
 983             g_list_free(probes);
 984             free(probe_key);
 985         }
 986 
 987         if (probe_complete_ops) {
 988             g_list_free(probe_complete_ops);
 989         }
 990 
 991         stop_op_key = stop_key(rsc);
 992         stop_ops = find_actions_exact(rsc->actions, stop_op_key, stop_node);
 993 
 994         for (local_gIter = stop_ops; local_gIter != NULL; local_gIter = local_gIter->next) {
 995             action_t *stop = (action_t *) local_gIter->data;
 996 
 997             if (is_set(stop->flags, pe_action_optional) == FALSE) {
 998                 stop_is_optional = FALSE;
 999             }
1000 
1001             if (is_set(stop->flags, pe_action_runnable) == FALSE) {
1002                 crm_debug("%s\t   %s (cancelled : stop un-runnable)",
1003                           crm_str(stop_node_uname), stopped_mon->uuid);
1004                 update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
1005             }
1006 
1007             if (is_set(rsc->flags, pe_rsc_managed)) {
1008                 custom_action_order(rsc, strdup(stop_op_key), stop,
1009                                     NULL, strdup(key), stopped_mon,
1010                                     pe_order_implies_then | pe_order_runnable_left, data_set);
1011             }
1012 
1013         }
1014 
1015         if (stop_ops) {
1016             g_list_free(stop_ops);
1017         }
1018         free(stop_op_key);
1019 
1020         if (is_optional == FALSE && probe_is_optional && stop_is_optional
1021             && is_set(rsc->flags, pe_rsc_managed) == FALSE) {
1022             pe_rsc_trace(rsc, "Marking %s optional on %s due to unmanaged",
1023                          key, crm_str(stop_node_uname));
1024             update_action_flags(stopped_mon, pe_action_optional, __FUNCTION__, __LINE__);
1025         }
1026 
1027         if (is_set(stopped_mon->flags, pe_action_optional)) {
1028             pe_rsc_trace(rsc, "%s\t   %s (optional)", crm_str(stop_node_uname), stopped_mon->uuid);
1029         }
1030 
1031         if (stop_node->details->online == FALSE || stop_node->details->unclean) {
1032             pe_rsc_debug(rsc, "%s\t   %s (cancelled : no node available)",
1033                          crm_str(stop_node_uname), stopped_mon->uuid);
1034             update_action_flags(stopped_mon, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
1035         }
1036 
1037         if (is_set(stopped_mon->flags, pe_action_runnable)
1038             && is_set(stopped_mon->flags, pe_action_optional) == FALSE) {
1039             crm_notice(" Start recurring %s (%llus) for %s on %s", stopped_mon->task,
1040                        interval_ms / 1000, rsc->id, crm_str(stop_node_uname));
1041         }
1042     }
1043 
1044     free(key);
1045 }
1046 
1047 void
1048 Recurring_Stopped(resource_t * rsc, action_t * start, node_t * node, pe_working_set_t * data_set)
     /*  */
1049 {
1050     if (is_not_set(rsc->flags, pe_rsc_maintenance) && 
1051         (node == NULL || node->details->maintenance == FALSE)) {
1052         xmlNode *operation = NULL;
1053 
1054         for (operation = __xml_first_child(rsc->ops_xml); operation != NULL;
1055              operation = __xml_next_element(operation)) {
1056             if (crm_str_eq((const char *)operation->name, "op", TRUE)) {
1057                 RecurringOp_Stopped(rsc, start, node, operation, data_set);
1058             }
1059         }
1060     }
1061 }
1062 
1063 static void
1064 handle_migration_actions(resource_t * rsc, node_t *current, node_t *chosen, pe_working_set_t * data_set)
     /*  */
1065 {
1066     action_t *migrate_to = NULL;
1067     action_t *migrate_from = NULL;
1068     action_t *start = NULL;
1069     action_t *stop = NULL;
1070     gboolean partial = rsc->partial_migration_target ? TRUE : FALSE;
1071 
1072     pe_rsc_trace(rsc, "Processing migration actions %s moving from %s to %s . partial migration = %s",
1073     rsc->id, current->details->id, chosen->details->id, partial ? "TRUE" : "FALSE");
1074     start = start_action(rsc, chosen, TRUE);
1075     stop = stop_action(rsc, current, TRUE);
1076 
1077     if (partial == FALSE) {
1078         migrate_to = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), RSC_MIGRATE, current, TRUE, TRUE, data_set);
1079     }
1080 
1081     migrate_from = custom_action(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), RSC_MIGRATED, chosen, TRUE, TRUE, data_set);
1082 
1083     if ((migrate_to && migrate_from) || (migrate_from && partial)) {
1084 
1085         set_bit(start->flags, pe_action_migrate_runnable);
1086         set_bit(stop->flags, pe_action_migrate_runnable);
1087 
1088         update_action_flags(start, pe_action_pseudo, __FUNCTION__, __LINE__);       /* easier than trying to delete it from the graph */
1089 
1090         /* order probes before migrations */
1091         if (partial) {
1092             set_bit(migrate_from->flags, pe_action_migrate_runnable);
1093             migrate_from->needs = start->needs;
1094 
1095             custom_action_order(rsc, generate_op_key(rsc->id, RSC_STATUS, 0), NULL,
1096                                 rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, pe_order_optional, data_set);
1097 
1098         } else {
1099             set_bit(migrate_from->flags, pe_action_migrate_runnable);
1100             set_bit(migrate_to->flags, pe_action_migrate_runnable);
1101             migrate_to->needs = start->needs;
1102 
1103             custom_action_order(rsc, generate_op_key(rsc->id, RSC_STATUS, 0), NULL,
1104                                 rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL, pe_order_optional, data_set);
1105             custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL,
1106                                 rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL, pe_order_optional | pe_order_implies_first_migratable, data_set);
1107         }
1108 
1109         custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL,
1110                             rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, pe_order_optional | pe_order_implies_first_migratable, data_set);
1111         custom_action_order(rsc, generate_op_key(rsc->id, RSC_MIGRATED, 0), NULL,
1112                             rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional | pe_order_implies_first_migratable | pe_order_pseudo_left, data_set);
1113 
1114     }
1115 
1116     if (migrate_to) {
1117         add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
1118         add_hash_param(migrate_to->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
1119 
1120         /* Pacemaker Remote connections don't require pending to be recorded in
1121          * the CIB. We can reduce CIB writes by not setting PENDING for them.
1122          */
1123         if (rsc->is_remote_node == FALSE) {
1124             /* migrate_to takes place on the source node, but can 
1125              * have an effect on the target node depending on how
1126              * the agent is written. Because of this, we have to maintain
1127              * a record that the migrate_to occurred, in case the source node
1128              * loses membership while the migrate_to action is still in-flight.
1129              */
1130             add_hash_param(migrate_to->meta, XML_OP_ATTR_PENDING, "true");
1131         }
1132     }
1133 
1134     if (migrate_from) {
1135         add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_SOURCE, current->details->uname);
1136         add_hash_param(migrate_from->meta, XML_LRM_ATTR_MIGRATE_TARGET, chosen->details->uname);
1137     }
1138 }
1139 
1140 void
1141 native_create_actions(resource_t * rsc, pe_working_set_t * data_set)
     /*  */
1142 {
1143     action_t *start = NULL;
1144     node_t *chosen = NULL;
1145     node_t *current = NULL;
1146     gboolean need_stop = FALSE;
1147     gboolean is_moving = FALSE;
1148     gboolean allow_migrate = is_set(rsc->flags, pe_rsc_allow_migrate) ? TRUE : FALSE;
1149 
1150     GListPtr gIter = NULL;
1151     int num_active_nodes = 0;
1152     enum rsc_role_e role = RSC_ROLE_UNKNOWN;
1153     enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
1154 
1155     CRM_ASSERT(rsc);
1156     chosen = rsc->allocated_to;
1157     if (chosen != NULL && rsc->next_role == RSC_ROLE_UNKNOWN) {
1158         rsc->next_role = RSC_ROLE_STARTED;
1159         pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));
1160 
1161     } else if (rsc->next_role == RSC_ROLE_UNKNOWN) {
1162         rsc->next_role = RSC_ROLE_STOPPED;
1163         pe_rsc_trace(rsc, "Fixed next_role: unknown -> %s", role2text(rsc->next_role));
1164     }
1165 
1166     pe_rsc_trace(rsc, "Processing state transition for %s %p: %s->%s", rsc->id, rsc,
1167                  role2text(rsc->role), role2text(rsc->next_role));
1168 
1169     if (rsc->running_on) {
1170         current = rsc->running_on->data;
1171     }
1172 
1173     for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
1174         node_t *n = (node_t *) gIter->data;
1175         if (rsc->partial_migration_source &&
1176             (n->details == rsc->partial_migration_source->details)) {
1177             current = rsc->partial_migration_source;
1178         }
1179         num_active_nodes++;
1180     }
1181 
1182     for (gIter = rsc->dangling_migrations; gIter != NULL; gIter = gIter->next) {
1183         node_t *current = (node_t *) gIter->data;
1184 
1185         action_t *stop = stop_action(rsc, current, FALSE);
1186 
1187         set_bit(stop->flags, pe_action_dangle);
1188         pe_rsc_trace(rsc, "Forcing a cleanup of %s on %s", rsc->id, current->details->uname);
1189 
1190         if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
1191             DeleteRsc(rsc, current, FALSE, data_set);
1192         }
1193     }
1194 
1195     if (num_active_nodes > 1) {
1196 
1197         if (num_active_nodes == 2
1198             && chosen
1199             && rsc->partial_migration_target
1200             && rsc->partial_migration_source
1201             && (current->details == rsc->partial_migration_source->details)
1202             && (chosen->details == rsc->partial_migration_target->details)) {
1203             /* Here the chosen node is still the migration target from a partial
1204              * migration. Attempt to continue the migration instead of recovering
1205              * by stopping the resource everywhere and starting it on a single node. */
1206             pe_rsc_trace(rsc,
1207                          "Will attempt to continue with a partial migration to target %s from %s",
1208                          rsc->partial_migration_target->details->id,
1209                          rsc->partial_migration_source->details->id);
1210         } else {
1211             const char *type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
1212             const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
1213 
1214             if(rsc->partial_migration_target && rsc->partial_migration_source) {
1215                 crm_notice("Resource %s can no longer migrate to %s. Stopping on %s too", rsc->id,
1216                            rsc->partial_migration_target->details->uname,
1217                            rsc->partial_migration_source->details->uname);
1218 
1219             } else {
1220                 pe_proc_err("Resource %s (%s::%s) is active on %d nodes %s",
1221                             rsc->id, class, type, num_active_nodes, recovery2text(rsc->recovery_type));
1222                 crm_warn("See %s for more information.",
1223                          "http://clusterlabs.org/wiki/FAQ#Resource_is_Too_Active");
1224             }
1225 
1226             if (rsc->recovery_type == recovery_stop_start) {
1227                 need_stop = TRUE;
1228             }
1229 
1230             /* If by chance a partial migration is in process,
1231              * but the migration target is not chosen still, clear all
1232              * partial migration data.  */
1233             rsc->partial_migration_source = rsc->partial_migration_target = NULL;
1234             allow_migrate = FALSE;
1235         }
1236     }
1237 
1238     if (is_set(rsc->flags, pe_rsc_start_pending)) {
1239         start = start_action(rsc, chosen, TRUE);
1240         set_bit(start->flags, pe_action_print_always);
1241     }
1242 
1243     if (current && chosen && current->details != chosen->details) {
1244         pe_rsc_trace(rsc, "Moving %s", rsc->id);
1245         is_moving = TRUE;
1246         need_stop = TRUE;
1247 
1248     } else if (is_set(rsc->flags, pe_rsc_failed)) {
1249         pe_rsc_trace(rsc, "Recovering %s", rsc->id);
1250         need_stop = TRUE;
1251 
1252     } else if (is_set(rsc->flags, pe_rsc_block)) {
1253         pe_rsc_trace(rsc, "Block %s", rsc->id);
1254         need_stop = TRUE;
1255 
1256     } else if (rsc->role > RSC_ROLE_STARTED && current != NULL && chosen != NULL) {
1257         /* Recovery of a promoted resource */
1258         start = start_action(rsc, chosen, TRUE);
1259         if (is_set(start->flags, pe_action_optional) == FALSE) {
1260             pe_rsc_trace(rsc, "Forced start %s", rsc->id);
1261             need_stop = TRUE;
1262         }
1263     }
1264 
1265     pe_rsc_trace(rsc, "Creating actions for %s: %s->%s", rsc->id,
1266                  role2text(rsc->role), role2text(rsc->next_role));
1267 
1268     /* Create any additional actions required when bringing resource down and
1269      * back up to same level.
1270      */
1271     role = rsc->role;
1272     while (role != RSC_ROLE_STOPPED) {
1273         next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
1274         pe_rsc_trace(rsc, "Down: Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
1275                      rsc->id, need_stop ? " required" : "");
1276         if (rsc_action_matrix[role][next_role] (rsc, current, !need_stop, data_set) == FALSE) {
1277             break;
1278         }
1279         role = next_role;
1280     }
1281 
1282 
1283     while (rsc->role <= rsc->next_role && role != rsc->role && is_not_set(rsc->flags, pe_rsc_block)) {
1284         next_role = rsc_state_matrix[role][rsc->role];
1285         pe_rsc_trace(rsc, "Up:   Executing: %s->%s (%s)%s", role2text(role), role2text(next_role),
1286                      rsc->id, need_stop ? " required" : "");
1287         if (rsc_action_matrix[role][next_role] (rsc, chosen, !need_stop, data_set) == FALSE) {
1288             break;
1289         }
1290         role = next_role;
1291     }
1292     role = rsc->role;
1293 
1294     /* Required steps from this role to the next */
1295     while (role != rsc->next_role) {
1296         next_role = rsc_state_matrix[role][rsc->next_role];
1297         pe_rsc_trace(rsc, "Role: Executing: %s->%s = (%s on %s)", role2text(role), role2text(next_role), rsc->id, chosen?chosen->details->uname:"NA");
1298         if (rsc_action_matrix[role][next_role] (rsc, chosen, FALSE, data_set) == FALSE) {
1299             break;
1300         }
1301         role = next_role;
1302     }
1303 
1304     if(is_set(rsc->flags, pe_rsc_block)) {
1305         pe_rsc_trace(rsc, "No monitor additional ops for blocked resource");
1306 
1307     } else if (rsc->next_role != RSC_ROLE_STOPPED || is_set(rsc->flags, pe_rsc_managed) == FALSE) {
1308         pe_rsc_trace(rsc, "Monitor ops for active resource");
1309         start = start_action(rsc, chosen, TRUE);
1310         Recurring(rsc, start, chosen, data_set);
1311         Recurring_Stopped(rsc, start, chosen, data_set);
1312     } else {
1313         pe_rsc_trace(rsc, "Monitor ops for in-active resource");
1314         Recurring_Stopped(rsc, NULL, NULL, data_set);
1315     }
1316 
1317     /* if we are stuck in a partial migration, where the target
1318      * of the partial migration no longer matches the chosen target.
1319      * A full stop/start is required */
1320     if (rsc->partial_migration_target && (chosen == NULL || rsc->partial_migration_target->details != chosen->details)) {
1321         pe_rsc_trace(rsc, "Not allowing partial migration to continue. %s", rsc->id);
1322         allow_migrate = FALSE;
1323 
1324     } else if (is_moving == FALSE ||
1325                is_not_set(rsc->flags, pe_rsc_managed) ||
1326                is_set(rsc->flags, pe_rsc_failed) ||
1327                is_set(rsc->flags, pe_rsc_start_pending) ||
1328                (current->details->unclean == TRUE) ||
1329                rsc->next_role < RSC_ROLE_STARTED) {
1330 
1331         allow_migrate = FALSE;
1332     }
1333 
1334     if (allow_migrate) {
1335         handle_migration_actions(rsc, current, chosen, data_set);
1336     }
1337 }
1338 
1339 static void
1340 rsc_avoids_remote_nodes(resource_t *rsc)
     /*  */
1341 {
1342     GHashTableIter iter;
1343     node_t *node = NULL;
1344     g_hash_table_iter_init(&iter, rsc->allowed_nodes);
1345     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
1346         if (node->details->remote_rsc) {
1347             node->weight = -INFINITY;
1348         }
1349     }
1350 }
1351 
1352 void
1353 native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
     /*  */
1354 {
1355     /* This function is on the critical path and worth optimizing as much as possible */
1356 
1357     resource_t *top = uber_parent(rsc);
1358     int type = pe_order_optional | pe_order_implies_then | pe_order_restart;
1359     gboolean is_stonith = is_set(rsc->flags, pe_rsc_fence_device);
1360 
1361     custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
1362                         rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, type, data_set);
1363 
1364     if (top->variant == pe_master || rsc->role > RSC_ROLE_SLAVE) {
1365         custom_action_order(rsc, generate_op_key(rsc->id, RSC_DEMOTE, 0), NULL,
1366                             rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
1367                             pe_order_implies_first_master, data_set);
1368 
1369         custom_action_order(rsc, generate_op_key(rsc->id, RSC_START, 0), NULL,
1370                             rsc, generate_op_key(rsc->id, RSC_PROMOTE, 0), NULL,
1371                             pe_order_runnable_left, data_set);
1372     }
1373 
1374     if (is_stonith == FALSE
1375         && is_set(data_set->flags, pe_flag_enable_unfencing)
1376         && is_set(rsc->flags, pe_rsc_needs_unfencing)) {
1377         /* Check if the node needs to be unfenced first */
1378         node_t *node = NULL;
1379         GHashTableIter iter;
1380 
1381         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
1382         while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
1383             action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set);
1384 
1385             crm_debug("Ordering any stops of %s before %s, and any starts after",
1386                       rsc->id, unfence->uuid);
1387 
1388             /*
1389              * It would be more efficient to order clone resources once,
1390              * rather than order each instance, but ordering the instance
1391              * allows us to avoid unnecessary dependencies that might conflict
1392              * with user constraints.
1393              *
1394              * @TODO: This constraint can still produce a transition loop if the
1395              * resource has a stop scheduled on the node being unfenced, and
1396              * there is a user ordering constraint to start some other resource
1397              * (which will be ordered after the unfence) before stopping this
1398              * resource. An example is "start some slow-starting cloned service
1399              * before stopping an associated virtual IP that may be moving to
1400              * it":
1401              *       stop this -> unfencing -> start that -> stop this
1402              */
1403             custom_action_order(rsc, stop_key(rsc), NULL,
1404                                 NULL, strdup(unfence->uuid), unfence,
1405                                 pe_order_optional|pe_order_same_node, data_set);
1406 
1407             custom_action_order(NULL, strdup(unfence->uuid), unfence,
1408                                 rsc, start_key(rsc), NULL,
1409                                 pe_order_implies_then_on_node|pe_order_same_node,
1410                                 data_set);
1411         }
1412     }
1413 
1414     if (is_not_set(rsc->flags, pe_rsc_managed)) {
1415         pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);
1416         return;
1417     }
1418 
1419     {
1420         action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
1421 
1422         custom_action_order(rsc, stop_key(rsc), NULL,
1423                             NULL, strdup(all_stopped->task), all_stopped,
1424                             pe_order_implies_then | pe_order_runnable_left, data_set);
1425     }
1426 
1427     if (g_hash_table_size(rsc->utilization) > 0
1428         && safe_str_neq(data_set->placement_strategy, "default")) {
1429         GHashTableIter iter;
1430         node_t *next = NULL;
1431         GListPtr gIter = NULL;
1432 
1433         pe_rsc_trace(rsc, "Creating utilization constraints for %s - strategy: %s",
1434                      rsc->id, data_set->placement_strategy);
1435 
1436         for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
1437             node_t *current = (node_t *) gIter->data;
1438 
1439             char *load_stopped_task = crm_concat(LOAD_STOPPED, current->details->uname, '_');
1440             action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
1441 
1442             if (load_stopped->node == NULL) {
1443                 load_stopped->node = node_copy(current);
1444                 update_action_flags(load_stopped, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
1445             }
1446 
1447             custom_action_order(rsc, stop_key(rsc), NULL,
1448                                 NULL, load_stopped_task, load_stopped, pe_order_load, data_set);
1449         }
1450 
1451         g_hash_table_iter_init(&iter, rsc->allowed_nodes);
1452         while (g_hash_table_iter_next(&iter, NULL, (void **)&next)) {
1453             char *load_stopped_task = crm_concat(LOAD_STOPPED, next->details->uname, '_');
1454             action_t *load_stopped = get_pseudo_op(load_stopped_task, data_set);
1455 
1456             if (load_stopped->node == NULL) {
1457                 load_stopped->node = node_copy(next);
1458                 update_action_flags(load_stopped, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
1459             }
1460 
1461             custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
1462                                 rsc, start_key(rsc), NULL, pe_order_load, data_set);
1463 
1464             custom_action_order(NULL, strdup(load_stopped_task), load_stopped,
1465                                 rsc, generate_op_key(rsc->id, RSC_MIGRATE, 0), NULL,
1466                                 pe_order_load, data_set);
1467 
1468             free(load_stopped_task);
1469         }
1470     }
1471 
1472     if (rsc->container) {
1473         resource_t *remote_rsc = NULL;
1474 
1475         /* A user can specify that a resource must start on a Pacemaker Remote
1476          * node by explicitly configuring it with the container=NODENAME
1477          * meta-attribute. This is of questionable merit, since location
1478          * constraints can accomplish the same thing. But we support it, so here
1479          * we check whether a resource (that is not itself a remote connection)
1480          * has container set to a remote node or guest node resource.
1481          */
1482         if (rsc->container->is_remote_node) {
1483             remote_rsc = rsc->container;
1484         } else if (rsc->is_remote_node == FALSE) {
1485             remote_rsc = rsc_contains_remote_node(data_set, rsc->container);
1486         }
1487 
1488         if (remote_rsc) {
1489             /* The container represents a Pacemaker Remote node, so force the
1490              * resource on the Pacemaker Remote node instead of colocating the
1491              * resource with the container resource.
1492              */
1493             GHashTableIter iter;
1494             node_t *node = NULL;
1495             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
1496             while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
1497                 if (node->details->remote_rsc != remote_rsc) {
1498                     node->weight = -INFINITY;
1499                 }
1500             }
1501         } else {
1502             /* This resource is either a filler for a container that does NOT
1503              * represent a Pacemaker Remote node, or a Pacemaker Remote
1504              * connection resource for a guest node or bundle.
1505              */
1506             int score;
1507 
1508             crm_trace("Order and colocate %s relative to its container %s",
1509                       rsc->id, rsc->container->id);
1510 
1511             custom_action_order(rsc->container, generate_op_key(rsc->container->id, RSC_START, 0), NULL,
1512                                 rsc, generate_op_key(rsc->id, RSC_START, 0), NULL,
1513                                 pe_order_implies_then | pe_order_runnable_left, data_set);
1514 
1515             custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
1516                                 rsc->container, generate_op_key(rsc->container->id, RSC_STOP, 0), NULL,
1517                                 pe_order_implies_first, data_set);
1518 
1519             if (is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
1520                 score = 10000;    /* Highly preferred but not essential */
1521             } else {
1522                 score = INFINITY; /* Force them to run on the same host */
1523             }
1524             rsc_colocation_new("resource-with-container", NULL, score, rsc,
1525                                rsc->container, NULL, NULL, data_set);
1526         }
1527     }
1528 
1529     if (rsc->is_remote_node || is_stonith) {
1530         /* don't allow remote nodes to run stonith devices
1531          * or remote connection resources.*/
1532         rsc_avoids_remote_nodes(rsc);
1533     }
1534 
1535     /* If this is a guest node's implicit remote connection, do not allow the
1536      * guest resource to live on a Pacemaker Remote node, to avoid nesting
1537      * remotes. However, allow bundles to run on remote nodes.
1538      */
1539     if (rsc->is_remote_node && rsc->container
1540         && is_not_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
1541         rsc_avoids_remote_nodes(rsc->container);
1542     }
1543 }
1544 
1545 void
1546 native_rsc_colocation_lh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
     /*  */
1547 {
1548     if (rsc_lh == NULL) {
1549         pe_err("rsc_lh was NULL for %s", constraint->id);
1550         return;
1551 
1552     } else if (constraint->rsc_rh == NULL) {
1553         pe_err("rsc_rh was NULL for %s", constraint->id);
1554         return;
1555     }
1556 
1557     pe_rsc_trace(rsc_lh, "Processing colocation constraint between %s and %s", rsc_lh->id,
1558                  rsc_rh->id);
1559 
1560     rsc_rh->cmds->rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
1561 }
1562 
1563 enum filter_colocation_res
1564 filter_colocation_constraint(resource_t * rsc_lh, resource_t * rsc_rh,
     /*  */
1565                              rsc_colocation_t * constraint, gboolean preview)
1566 {
1567     if (constraint->score == 0) {
1568         return influence_nothing;
1569     }
1570 
1571     /* rh side must be allocated before we can process constraint */
1572     if (preview == FALSE && is_set(rsc_rh->flags, pe_rsc_provisional)) {
1573         return influence_nothing;
1574     }
1575 
1576     if ((constraint->role_lh >= RSC_ROLE_SLAVE) &&
1577         rsc_lh->parent &&
1578         rsc_lh->parent->variant == pe_master && is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
1579 
1580         /* LH and RH resources have already been allocated, place the correct
1581          * priority oh LH rsc for the given multistate resource role */
1582         return influence_rsc_priority;
1583     }
1584 
1585     if (preview == FALSE && is_not_set(rsc_lh->flags, pe_rsc_provisional)) {
1586         /* error check */
1587         struct node_shared_s *details_lh;
1588         struct node_shared_s *details_rh;
1589 
1590         if ((constraint->score > -INFINITY) && (constraint->score < INFINITY)) {
1591             return influence_nothing;
1592         }
1593 
1594         details_rh = rsc_rh->allocated_to ? rsc_rh->allocated_to->details : NULL;
1595         details_lh = rsc_lh->allocated_to ? rsc_lh->allocated_to->details : NULL;
1596 
1597         if (constraint->score == INFINITY && details_lh != details_rh) {
1598             crm_err("%s and %s are both allocated"
1599                     " but to different nodes: %s vs. %s",
1600                     rsc_lh->id, rsc_rh->id,
1601                     details_lh ? details_lh->uname : "n/a", details_rh ? details_rh->uname : "n/a");
1602 
1603         } else if (constraint->score == -INFINITY && details_lh == details_rh) {
1604             crm_err("%s and %s are both allocated"
1605                     " but to the SAME node: %s",
1606                     rsc_lh->id, rsc_rh->id, details_rh ? details_rh->uname : "n/a");
1607         }
1608 
1609         return influence_nothing;
1610     }
1611 
1612     if (constraint->score > 0
1613         && constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh != rsc_lh->next_role) {
1614         crm_trace("LH: Skipping constraint: \"%s\" state filter nextrole is %s",
1615                   role2text(constraint->role_lh), role2text(rsc_lh->next_role));
1616         return influence_nothing;
1617     }
1618 
1619     if (constraint->score > 0
1620         && constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh != rsc_rh->next_role) {
1621         crm_trace("RH: Skipping constraint: \"%s\" state filter", role2text(constraint->role_rh));
1622         return FALSE;
1623     }
1624 
1625     if (constraint->score < 0
1626         && constraint->role_lh != RSC_ROLE_UNKNOWN && constraint->role_lh == rsc_lh->next_role) {
1627         crm_trace("LH: Skipping -ve constraint: \"%s\" state filter",
1628                   role2text(constraint->role_lh));
1629         return influence_nothing;
1630     }
1631 
1632     if (constraint->score < 0
1633         && constraint->role_rh != RSC_ROLE_UNKNOWN && constraint->role_rh == rsc_rh->next_role) {
1634         crm_trace("RH: Skipping -ve constraint: \"%s\" state filter",
1635                   role2text(constraint->role_rh));
1636         return influence_nothing;
1637     }
1638 
1639     return influence_rsc_location;
1640 }
1641 
1642 static void
1643 influence_priority(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
     /*  */
1644 {
1645     const char *rh_value = NULL;
1646     const char *lh_value = NULL;
1647     const char *attribute = CRM_ATTR_ID;
1648     int score_multiplier = 1;
1649 
1650     if (constraint->node_attribute != NULL) {
1651         attribute = constraint->node_attribute;
1652     }
1653 
1654     if (!rsc_rh->allocated_to || !rsc_lh->allocated_to) {
1655         return;
1656     }
1657 
1658     lh_value = pe_node_attribute_raw(rsc_lh->allocated_to, attribute);
1659     rh_value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);
1660 
1661     if (!safe_str_eq(lh_value, rh_value)) {
1662         if(constraint->score == INFINITY && constraint->role_lh == RSC_ROLE_MASTER) {
1663             rsc_lh->priority = -INFINITY;
1664         }
1665         return;
1666     }
1667 
1668     if (constraint->role_rh && (constraint->role_rh != rsc_rh->next_role)) {
1669         return;
1670     }
1671 
1672     if (constraint->role_lh == RSC_ROLE_SLAVE) {
1673         score_multiplier = -1;
1674     }
1675 
1676     rsc_lh->priority = merge_weights(score_multiplier * constraint->score, rsc_lh->priority);
1677 }
1678 
1679 static void
1680 colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
     /*  */
1681 {
1682     const char *tmp = NULL;
1683     const char *value = NULL;
1684     const char *attribute = CRM_ATTR_ID;
1685 
1686     GHashTable *work = NULL;
1687     gboolean do_check = FALSE;
1688 
1689     GHashTableIter iter;
1690     node_t *node = NULL;
1691 
1692     if (constraint->node_attribute != NULL) {
1693         attribute = constraint->node_attribute;
1694     }
1695 
1696     if (rsc_rh->allocated_to) {
1697         value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute);
1698         do_check = TRUE;
1699 
1700     } else if (constraint->score < 0) {
1701         /* nothing to do:
1702          *   anti-colocation with something that is not running
1703          */
1704         return;
1705     }
1706 
1707     work = node_hash_dup(rsc_lh->allowed_nodes);
1708 
1709     g_hash_table_iter_init(&iter, work);
1710     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
1711         tmp = pe_node_attribute_raw(node, attribute);
1712         if (do_check && safe_str_eq(tmp, value)) {
1713             if (constraint->score < INFINITY) {
1714                 pe_rsc_trace(rsc_lh, "%s: %s.%s += %d", constraint->id, rsc_lh->id,
1715                              node->details->uname, constraint->score);
1716                 node->weight = merge_weights(constraint->score, node->weight);
1717             }
1718 
1719         } else if (do_check == FALSE || constraint->score >= INFINITY) {
1720             pe_rsc_trace(rsc_lh, "%s: %s.%s -= %d (%s)", constraint->id, rsc_lh->id,
1721                          node->details->uname, constraint->score,
1722                          do_check ? "failed" : "unallocated");
1723             node->weight = merge_weights(-constraint->score, node->weight);
1724         }
1725     }
1726 
1727     if (can_run_any(work)
1728         || constraint->score <= -INFINITY || constraint->score >= INFINITY) {
1729         g_hash_table_destroy(rsc_lh->allowed_nodes);
1730         rsc_lh->allowed_nodes = work;
1731         work = NULL;
1732 
1733     } else {
1734         static char score[33];
1735 
1736         score2char_stack(constraint->score, score, sizeof(score));
1737 
1738         pe_rsc_info(rsc_lh, "%s: Rolling back scores from %s (%d, %s)",
1739                     rsc_lh->id, rsc_rh->id, do_check, score);
1740     }
1741 
1742     if (work) {
1743         g_hash_table_destroy(work);
1744     }
1745 }
1746 
1747 void
1748 native_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
     /*  */
1749 {
1750     enum filter_colocation_res filter_results;
1751 
1752     CRM_ASSERT(rsc_lh);
1753     CRM_ASSERT(rsc_rh);
1754     filter_results = filter_colocation_constraint(rsc_lh, rsc_rh, constraint, FALSE);
1755     pe_rsc_trace(rsc_lh, "%sColocating %s with %s (%s, weight=%d, filter=%d)",
1756                  constraint->score >= 0 ? "" : "Anti-",
1757                  rsc_lh->id, rsc_rh->id, constraint->id, constraint->score, filter_results);
1758 
1759     switch (filter_results) {
1760         case influence_rsc_priority:
1761             influence_priority(rsc_lh, rsc_rh, constraint);
1762             break;
1763         case influence_rsc_location:
1764             pe_rsc_trace(rsc_lh, "%sColocating %s with %s (%s, weight=%d)",
1765                          constraint->score >= 0 ? "" : "Anti-",
1766                          rsc_lh->id, rsc_rh->id, constraint->id, constraint->score);
1767             colocation_match(rsc_lh, rsc_rh, constraint);
1768             break;
1769         case influence_nothing:
1770         default:
1771             return;
1772     }
1773 }
1774 
1775 static gboolean
1776 filter_rsc_ticket(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket)
     /*  */
1777 {
1778     if (rsc_ticket->role_lh != RSC_ROLE_UNKNOWN && rsc_ticket->role_lh != rsc_lh->role) {
1779         pe_rsc_trace(rsc_lh, "LH: Skipping constraint: \"%s\" state filter",
1780                      role2text(rsc_ticket->role_lh));
1781         return FALSE;
1782     }
1783 
1784     return TRUE;
1785 }
1786 
1787 void
1788 rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working_set_t * data_set)
     /*  */
1789 {
1790     if (rsc_ticket == NULL) {
1791         pe_err("rsc_ticket was NULL");
1792         return;
1793     }
1794 
1795     if (rsc_lh == NULL) {
1796         pe_err("rsc_lh was NULL for %s", rsc_ticket->id);
1797         return;
1798     }
1799 
1800     if (rsc_ticket->ticket->granted && rsc_ticket->ticket->standby == FALSE) {
1801         return;
1802     }
1803 
1804     if (rsc_lh->children) {
1805         GListPtr gIter = rsc_lh->children;
1806 
1807         pe_rsc_trace(rsc_lh, "Processing ticket dependencies from %s", rsc_lh->id);
1808 
1809         for (; gIter != NULL; gIter = gIter->next) {
1810             resource_t *child_rsc = (resource_t *) gIter->data;
1811 
1812             rsc_ticket_constraint(child_rsc, rsc_ticket, data_set);
1813         }
1814         return;
1815     }
1816 
1817     pe_rsc_trace(rsc_lh, "%s: Processing ticket dependency on %s (%s, %s)",
1818                  rsc_lh->id, rsc_ticket->ticket->id, rsc_ticket->id,
1819                  role2text(rsc_ticket->role_lh));
1820 
1821     if (rsc_ticket->ticket->granted == FALSE && g_list_length(rsc_lh->running_on) > 0) {
1822         GListPtr gIter = NULL;
1823 
1824         switch (rsc_ticket->loss_policy) {
1825             case loss_ticket_stop:
1826                 resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
1827                 break;
1828 
1829             case loss_ticket_demote:
1830                 /*Promotion score will be set to -INFINITY in master_promotion_order() */
1831                 if (rsc_ticket->role_lh != RSC_ROLE_MASTER) {
1832                     resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
1833                 }
1834                 break;
1835 
1836             case loss_ticket_fence:
1837                 if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
1838                     return;
1839                 }
1840 
1841                 resource_location(rsc_lh, NULL, -INFINITY, "__loss_of_ticket__", data_set);
1842 
1843                 for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
1844                     node_t *node = (node_t *) gIter->data;
1845 
1846                     pe_fence_node(data_set, node, "deadman ticket was lost");
1847                 }
1848                 break;
1849 
1850             case loss_ticket_freeze:
1851                 if (filter_rsc_ticket(rsc_lh, rsc_ticket) == FALSE) {
1852                     return;
1853                 }
1854                 if (g_list_length(rsc_lh->running_on) > 0) {
1855                     clear_bit(rsc_lh->flags, pe_rsc_managed);
1856                     set_bit(rsc_lh->flags, pe_rsc_block);
1857                 }
1858                 break;
1859         }
1860 
1861     } else if (rsc_ticket->ticket->granted == FALSE) {
1862 
1863         if (rsc_ticket->role_lh != RSC_ROLE_MASTER || rsc_ticket->loss_policy == loss_ticket_stop) {
1864             resource_location(rsc_lh, NULL, -INFINITY, "__no_ticket__", data_set);
1865         }
1866 
1867     } else if (rsc_ticket->ticket->standby) {
1868 
1869         if (rsc_ticket->role_lh != RSC_ROLE_MASTER || rsc_ticket->loss_policy == loss_ticket_stop) {
1870             resource_location(rsc_lh, NULL, -INFINITY, "__ticket_standby__", data_set);
1871         }
1872     }
1873 }
1874 
1875 enum pe_action_flags
1876 native_action_flags(action_t * action, node_t * node)
     /*  */
1877 {
1878     return action->flags;
1879 }
1880 
1881 enum pe_graph_flags
1882 native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
     /*  */
1883                       enum pe_action_flags filter, enum pe_ordering type)
1884 {
1885     /* flags == get_action_flags(first, then_node) called from update_action() */
1886     enum pe_graph_flags changed = pe_graph_none;
1887     enum pe_action_flags then_flags = then->flags;
1888     enum pe_action_flags first_flags = first->flags;
1889 
1890     crm_trace(   "Testing %s on %s (0x%.6x) with %s 0x%.6x",
1891                  first->uuid, first->node ? first->node->details->uname : "[none]",
1892                  first->flags, then->uuid, then->flags);
1893 
1894     if (type & pe_order_asymmetrical) {
1895         resource_t *then_rsc = then->rsc;
1896         enum rsc_role_e then_rsc_role = then_rsc ? then_rsc->fns->state(then_rsc, TRUE) : 0;
1897 
1898         if (!then_rsc) {
1899             /* ignore */
1900         } else if ((then_rsc_role == RSC_ROLE_STOPPED) && safe_str_eq(then->task, RSC_STOP)) {
1901             /* ignore... if 'then' is supposed to be stopped after 'first', but
1902              * then is already stopped, there is nothing to be done when non-symmetrical.  */
1903         } else if ((then_rsc_role >= RSC_ROLE_STARTED)
1904                    && safe_str_eq(then->task, RSC_START)
1905                    && then->node
1906                    && then_rsc->running_on
1907                    && g_list_length(then_rsc->running_on) == 1
1908                    && then->node->details == ((node_t *) then_rsc->running_on->data)->details) {
1909             /* ignore... if 'then' is supposed to be started after 'first', but
1910              * then is already started, there is nothing to be done when non-symmetrical.  */
1911         } else if (!(first->flags & pe_action_runnable)) {
1912             /* prevent 'then' action from happening if 'first' is not runnable and
1913              * 'then' has not yet occurred. */
1914             pe_action_implies(then, first, pe_action_optional);
1915             pe_action_implies(then, first, pe_action_runnable);
1916 
1917             pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid);
1918         } else {
1919             /* ignore... then is allowed to start/stop if it wants to. */
1920         }
1921     }
1922 
1923     if (type & pe_order_implies_first) {
1924         if (is_set(filter, pe_action_optional) && is_not_set(flags /* Should be then_flags? */, pe_action_optional)) {
1925             // Needs is_set(first_flags, pe_action_optional) too?
1926             pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
1927             pe_action_implies(first, then, pe_action_optional);
1928         }
1929 
1930         if (is_set(flags, pe_action_migrate_runnable) &&
1931             is_set(then->flags, pe_action_migrate_runnable) == FALSE &&
1932             is_set(then->flags, pe_action_optional) == FALSE) {
1933 
1934             pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s",
1935                          first->uuid, then->uuid);
1936             pe_action_implies(first, then, pe_action_migrate_runnable);
1937         }
1938     }
1939 
1940     if (type & pe_order_implies_first_master) {
1941         if ((filter & pe_action_optional) &&
1942             ((then->flags & pe_action_optional) == FALSE) &&
1943             then->rsc && (then->rsc->role == RSC_ROLE_MASTER)) {
1944             pe_action_implies(first, then, pe_action_optional);
1945 
1946             if (is_set(first->flags, pe_action_migrate_runnable) &&
1947                 is_set(then->flags, pe_action_migrate_runnable) == FALSE) {
1948 
1949                 pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid);
1950                 pe_action_implies(first, then, pe_action_migrate_runnable);
1951             }
1952             pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid);
1953         }
1954     }
1955 
1956     if ((type & pe_order_implies_first_migratable)
1957         && is_set(filter, pe_action_optional)) {
1958 
1959         if (((then->flags & pe_action_migrate_runnable) == FALSE) ||
1960             ((then->flags & pe_action_runnable) == FALSE)) {
1961 
1962             pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid);
1963             pe_action_implies(first, then, pe_action_runnable);
1964         }
1965 
1966         if ((then->flags & pe_action_optional) == 0) {
1967             pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid);
1968             pe_action_implies(first, then, pe_action_optional);
1969         }
1970     }
1971 
1972     if ((type & pe_order_pseudo_left)
1973         && is_set(filter, pe_action_optional)) {
1974 
1975         if ((first->flags & pe_action_runnable) == FALSE) {
1976             pe_action_implies(then, first, pe_action_migrate_runnable);
1977             pe_clear_action_bit(then, pe_action_pseudo);
1978             pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid);
1979         }
1980 
1981     }
1982 
1983     if (is_set(type, pe_order_runnable_left)
1984         && is_set(filter, pe_action_runnable)
1985         && is_set(then->flags, pe_action_runnable)
1986         && is_set(flags, pe_action_runnable) == FALSE) {
1987         pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid);
1988         pe_action_implies(then, first, pe_action_runnable);
1989         pe_action_implies(then, first, pe_action_migrate_runnable);
1990     }
1991 
1992     if (is_set(type, pe_order_implies_then)
1993         && is_set(filter, pe_action_optional)
1994         && is_set(then->flags, pe_action_optional)
1995         && is_set(flags, pe_action_optional) == FALSE) {
1996 
1997         /* in this case, treat migrate_runnable as if first is optional */
1998         if (is_set(first->flags, pe_action_migrate_runnable) == FALSE) {
1999            pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid);
2000            pe_action_implies(then, first, pe_action_optional);
2001         }
2002     }
2003 
2004     if (is_set(type, pe_order_restart)) {
2005         const char *reason = NULL;
2006 
2007         CRM_ASSERT(first->rsc && first->rsc->variant == pe_native);
2008         CRM_ASSERT(then->rsc && then->rsc->variant == pe_native);
2009 
2010         if ((filter & pe_action_runnable)
2011             && (then->flags & pe_action_runnable) == 0
2012             && (then->rsc->flags & pe_rsc_managed)) {
2013             reason = "shutdown";
2014         }
2015 
2016         if ((filter & pe_action_optional) && (then->flags & pe_action_optional) == 0) {
2017             reason = "recover";
2018         }
2019 
2020         if (reason && is_set(first->flags, pe_action_optional)) {
2021             if (is_set(first->flags, pe_action_runnable)
2022                 || is_not_set(then->flags, pe_action_optional)) {
2023                 pe_rsc_trace(first->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid);
2024                 pe_action_implies(first, then, pe_action_optional);
2025             }
2026         }
2027 
2028         if (reason && is_not_set(first->flags, pe_action_optional)
2029             && is_not_set(first->flags, pe_action_runnable)) {
2030             pe_rsc_trace(then->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid);
2031             pe_action_implies(then, first, pe_action_runnable);
2032         }
2033 
2034         if (reason &&
2035             is_not_set(first->flags, pe_action_optional) &&
2036             is_set(first->flags, pe_action_migrate_runnable)  &&
2037             is_not_set(then->flags, pe_action_migrate_runnable)) {
2038 
2039             pe_action_implies(first, then, pe_action_migrate_runnable);
2040         }
2041 
2042     }
2043 
2044     if (then_flags != then->flags) {
2045         changed |= pe_graph_updated_then;
2046         pe_rsc_trace(then->rsc,
2047                      "Then: Flags for %s on %s are now  0x%.6x (was 0x%.6x) because of %s 0x%.6x",
2048                      then->uuid, then->node ? then->node->details->uname : "[none]", then->flags,
2049                      then_flags, first->uuid, first->flags);
2050 
2051         if(then->rsc && then->rsc->parent) {
2052             /* "X_stop then X_start" doesn't get handled for cloned groups unless we do this */
2053             update_action(then);
2054         }
2055     }
2056 
2057     if (first_flags != first->flags) {
2058         changed |= pe_graph_updated_first;
2059         pe_rsc_trace(first->rsc,
2060                      "First: Flags for %s on %s are now  0x%.6x (was 0x%.6x) because of %s 0x%.6x",
2061                      first->uuid, first->node ? first->node->details->uname : "[none]",
2062                      first->flags, first_flags, then->uuid, then->flags);
2063     }
2064 
2065     return changed;
2066 }
2067 
2068 void
2069 native_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
     /*  */
2070 {
2071     GListPtr gIter = NULL;
2072     GHashTableIter iter;
2073     node_t *node = NULL;
2074 
2075     if (constraint == NULL) {
2076         pe_err("Constraint is NULL");
2077         return;
2078 
2079     } else if (rsc == NULL) {
2080         pe_err("LHS of rsc_to_node (%s) is NULL", constraint->id);
2081         return;
2082     }
2083 
2084     pe_rsc_trace(rsc, "Applying %s (%s) to %s", constraint->id,
2085                  role2text(constraint->role_filter), rsc->id);
2086 
2087     /* take "lifetime" into account */
2088     if (constraint->role_filter > RSC_ROLE_UNKNOWN && constraint->role_filter != rsc->next_role) {
2089         pe_rsc_debug(rsc, "Constraint (%s) is not active (role : %s vs. %s)",
2090                      constraint->id, role2text(constraint->role_filter), role2text(rsc->next_role));
2091         return;
2092 
2093     } else if (is_active(constraint) == FALSE) {
2094         pe_rsc_trace(rsc, "Constraint (%s) is not active", constraint->id);
2095         return;
2096     }
2097 
2098     if (constraint->node_list_rh == NULL) {
2099         pe_rsc_trace(rsc, "RHS of constraint %s is NULL", constraint->id);
2100         return;
2101     }
2102 
2103     for (gIter = constraint->node_list_rh; gIter != NULL; gIter = gIter->next) {
2104         node_t *node = (node_t *) gIter->data;
2105         node_t *other_node = NULL;
2106 
2107         other_node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
2108 
2109         if (other_node != NULL) {
2110             pe_rsc_trace(rsc, "%s + %s: %d + %d",
2111                          node->details->uname,
2112                          other_node->details->uname, node->weight, other_node->weight);
2113             other_node->weight = merge_weights(other_node->weight, node->weight);
2114 
2115         } else {
2116             other_node = node_copy(node);
2117 
2118             pe_rsc_trace(rsc, "%s: %d (insert %d)", other_node->details->uname, other_node->weight, constraint->discover_mode);
2119             g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node);
2120         }
2121 
2122         if (other_node->rsc_discover_mode < constraint->discover_mode) {
2123             if (constraint->discover_mode == pe_discover_exclusive) {
2124                 rsc->exclusive_discover = TRUE;
2125             }
2126             /* exclusive > never > always... always is default */
2127             other_node->rsc_discover_mode = constraint->discover_mode;
2128         }
2129     }
2130 
2131     g_hash_table_iter_init(&iter, rsc->allowed_nodes);
2132     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
2133         pe_rsc_trace(rsc, "%s + %s : %d", rsc->id, node->details->uname, node->weight);
2134     }
2135 }
2136 
2137 void
2138 native_expand(resource_t * rsc, pe_working_set_t * data_set)
     /*  */
2139 {
2140     GListPtr gIter = NULL;
2141 
2142     CRM_ASSERT(rsc);
2143     pe_rsc_trace(rsc, "Processing actions from %s", rsc->id);
2144 
2145     for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
2146         action_t *action = (action_t *) gIter->data;
2147 
2148         crm_trace("processing action %d for rsc=%s", action->id, rsc->id);
2149         graph_element_from_action(action, data_set);
2150     }
2151 
2152     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
2153         resource_t *child_rsc = (resource_t *) gIter->data;
2154 
2155         child_rsc->cmds->expand(child_rsc, data_set);
2156     }
2157 }
2158 
2159 #define log_change(a, fmt, args...)  do {                         \
2160         if(a && a->reason && terminal) {                          \
2161             printf(" * "fmt" \tdue to %s\n", ##args, a->reason);    \
2162         } else if(a && a->reason) {                               \
2163             crm_notice(fmt" \tdue to %s", ##args, a->reason);       \
2164         } else if(terminal) {                                     \
2165             printf(" * "fmt"\n", ##args);                         \
2166         } else {                                                  \
2167             crm_notice(fmt, ##args);                              \
2168         }                                                         \
2169     } while(0)
2170 
2171 #define STOP_SANITY_ASSERT(lineno) do {                                 \
2172         if(current && current->details->unclean) {                      \
2173             /* It will be a pseudo op */                                \
2174         } else if(stop == NULL) {                                       \
2175             crm_err("%s:%d: No stop action exists for %s", __FUNCTION__, lineno, rsc->id); \
2176             CRM_ASSERT(stop != NULL);                                   \
2177         } else if(is_set(stop->flags, pe_action_optional)) { \
2178             crm_err("%s:%d: Action %s is still optional", __FUNCTION__, lineno, stop->uuid); \
2179             CRM_ASSERT(is_not_set(stop->flags, pe_action_optional));    \
2180         }                                                               \
2181     } while(0)
2182 
2183 static int rsc_width = 5;
2184 static int detail_width = 5;
2185 static void
2186 LogAction(const char *change, resource_t *rsc, pe_node_t *origin, pe_node_t *destination, pe_action_t *action, pe_action_t *source, gboolean terminal)
     /*  */
2187 {
2188     int len = 0;
2189     char *reason = NULL;
2190     char *details = NULL;
2191     bool same_host = FALSE;
2192     bool same_role = FALSE;
2193     bool need_role = FALSE;
2194 
2195     CRM_ASSERT(action);
2196     CRM_ASSERT(destination != NULL || origin != NULL);
2197 
2198     if(source == NULL) {
2199         source = action;
2200     }
2201 
2202     len = strlen(rsc->id);
2203     if(len > rsc_width) {
2204         rsc_width = len + 2;
2205     }
2206 
2207     if(rsc->role > RSC_ROLE_STARTED || rsc->next_role > RSC_ROLE_SLAVE) {
2208         need_role = TRUE;
2209     }
2210 
2211     if(origin != NULL && destination != NULL && origin->details == destination->details) {
2212         same_host = TRUE;
2213     }
2214 
2215     if(rsc->role == rsc->next_role) {
2216         same_role = TRUE;
2217     }
2218 
2219     if(need_role && origin == NULL) {
2220         /* Promoting from Stopped */
2221         details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), destination->details->uname);
2222 
2223     } else if(need_role && destination == NULL) {
2224         /* Demoting a Master or Stopping a Slave */
2225         details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);
2226 
2227     } else if(origin == NULL || destination == NULL) {
2228         /* Starting or stopping a resource */
2229         details = crm_strdup_printf("%s", origin?origin->details->uname:destination->details->uname);
2230 
2231     } else if(need_role && same_role && same_host) {
2232         /* Recovering or Restarting a Master/Slave resource */
2233         details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname);
2234 
2235     } else if(same_role && same_host) {
2236         /* Recovering or Restarting a normal resource */
2237         details = crm_strdup_printf("%s", origin->details->uname);
2238 
2239     } else if(same_role && need_role) {
2240         /* Moving a Master/Slave resource */
2241         details = crm_strdup_printf("%s -> %s %s", origin->details->uname, destination->details->uname, role2text(rsc->role));
2242 
2243     } else if(same_role) {
2244         /* Moving a normal resource */
2245         details = crm_strdup_printf("%s -> %s", origin->details->uname, destination->details->uname);
2246 
2247     } else if(same_host) {
2248         /* Promoting or Demoting a Master/Slave resource */
2249         details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), origin->details->uname);
2250 
2251     } else {
2252         /* Moving and promoting/demoting */
2253         details = crm_strdup_printf("%s %s -> %s %s", role2text(rsc->role), origin->details->uname, role2text(rsc->next_role), destination->details->uname);
2254     }
2255 
2256     len = strlen(details);
2257     if(len > detail_width) {
2258         detail_width = len;
2259     }
2260 
2261     if(source->reason && is_not_set(action->flags, pe_action_runnable)) {
2262         reason = crm_strdup_printf(" due to %s (blocked)", source->reason);
2263 
2264     } else if(source->reason) {
2265         reason = crm_strdup_printf(" due to %s", source->reason);
2266 
2267     } else if(is_not_set(action->flags, pe_action_runnable)) {
2268         reason = strdup(" blocked");
2269 
2270     } else {
2271         reason = strdup("");
2272     }
2273 
2274     if(terminal) {
2275         printf(" * %-8s   %-*s   ( %*s )  %s\n", change, rsc_width, rsc->id, detail_width, details, reason);
2276     } else {
2277         crm_notice(" * %-8s   %-*s   ( %*s )  %s", change, rsc_width, rsc->id, detail_width, details, reason);
2278     }
2279 
2280     free(details);
2281     free(reason);
2282 }
2283 
2284 
2285 void
2286 LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
     /*  */
2287 {
2288     node_t *next = NULL;
2289     node_t *current = NULL;
2290 
2291     action_t *stop = NULL;
2292     action_t *start = NULL;
2293     action_t *demote = NULL;
2294     action_t *promote = NULL;
2295 
2296     char *key = NULL;
2297     gboolean moving = FALSE;
2298     GListPtr possible_matches = NULL;
2299 
2300     if(rsc->variant == pe_container) {
2301         container_LogActions(rsc, data_set, terminal);
2302         return;
2303     }
2304 
2305     if (rsc->children) {
2306         GListPtr gIter = NULL;
2307 
2308         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
2309             resource_t *child_rsc = (resource_t *) gIter->data;
2310 
2311             LogActions(child_rsc, data_set, terminal);
2312         }
2313         return;
2314     }
2315 
2316     next = rsc->allocated_to;
2317     if (rsc->running_on) {
2318         if (g_list_length(rsc->running_on) > 1 && rsc->partial_migration_source) {
2319             current = rsc->partial_migration_source;
2320         } else {
2321             current = rsc->running_on->data;
2322         }
2323 
2324         if (rsc->role == RSC_ROLE_STOPPED) {
2325             /*
2326              * This can occur when resources are being recovered
2327              * We fiddle with the current role in native_create_actions()
2328              */
2329             rsc->role = RSC_ROLE_STARTED;
2330         }
2331     }
2332 
2333     if (current == NULL && is_set(rsc->flags, pe_rsc_orphan)) {
2334         /* Don't log stopped orphans */
2335         return;
2336     }
2337 
2338     if (is_not_set(rsc->flags, pe_rsc_managed)
2339         || (current == NULL && next == NULL)) {
2340         pe_rsc_info(rsc, "Leave   %s\t(%s%s)",
2341                     rsc->id, role2text(rsc->role), is_not_set(rsc->flags,
2342                                                               pe_rsc_managed) ? " unmanaged" : "");
2343         return;
2344     }
2345 
2346     if (current != NULL && next != NULL && safe_str_neq(current->details->id, next->details->id)) {
2347         moving = TRUE;
2348     }
2349 
2350     key = start_key(rsc);
2351     possible_matches = find_actions(rsc->actions, key, next);
2352     free(key);
2353     if (possible_matches) {
2354         start = possible_matches->data;
2355         g_list_free(possible_matches);
2356     }
2357 
2358     key = stop_key(rsc);
2359     if(start == NULL || is_set(start->flags, pe_action_runnable) == FALSE) {
2360         possible_matches = find_actions(rsc->actions, key, NULL);
2361     } else {
2362         possible_matches = find_actions(rsc->actions, key, current);
2363     }
2364     if (possible_matches) {
2365         stop = possible_matches->data;
2366         g_list_free(possible_matches);
2367     }
2368     free(key);
2369 
2370     key = promote_key(rsc);
2371     possible_matches = find_actions(rsc->actions, key, next);
2372     free(key);
2373     if (possible_matches) {
2374         promote = possible_matches->data;
2375         g_list_free(possible_matches);
2376     }
2377 
2378     key = demote_key(rsc);
2379     possible_matches = find_actions(rsc->actions, key, next);
2380     free(key);
2381     if (possible_matches) {
2382         demote = possible_matches->data;
2383         g_list_free(possible_matches);
2384     }
2385 
2386     if (rsc->role == rsc->next_role) {
2387         action_t *migrate_op = NULL;
2388 
2389         key = generate_op_key(rsc->id, RSC_MIGRATED, 0);
2390         possible_matches = find_actions(rsc->actions, key, next);
2391         free(key);
2392 
2393         if (possible_matches) {
2394             migrate_op = possible_matches->data;
2395         }
2396 
2397         CRM_CHECK(next != NULL,);
2398         if (next == NULL) {
2399         } else if (migrate_op && is_set(migrate_op->flags, pe_action_runnable) && current) {
2400             LogAction("Migrate", rsc, current, next, start, NULL, terminal);
2401 
2402         } else if (is_set(rsc->flags, pe_rsc_reload)) {
2403             LogAction("Reload", rsc, current, next, start, NULL, terminal);
2404 
2405         } else if (start == NULL || is_set(start->flags, pe_action_optional)) {
2406             pe_rsc_info(rsc, "Leave   %s\t(%s %s)", rsc->id, role2text(rsc->role),
2407                         next->details->uname);
2408 
2409         } else if (start && is_set(start->flags, pe_action_runnable) == FALSE) {
2410             LogAction("Stop", rsc, current, NULL, stop,
2411                       (stop && stop->reason)? stop : start, terminal);
2412             STOP_SANITY_ASSERT(__LINE__);
2413 
2414         } else if (moving && current) {
2415             LogAction(is_set(rsc->flags, pe_rsc_failed) ? "Recover" : "Move",
2416                       rsc, current, next, stop, NULL, terminal);
2417 
2418         } else if (is_set(rsc->flags, pe_rsc_failed)) {
2419             LogAction("Recover", rsc, current, NULL, stop, NULL, terminal);
2420             STOP_SANITY_ASSERT(__LINE__);
2421 
2422         } else {
2423             LogAction("Restart", rsc, current, next, start, NULL, terminal);
2424             /* STOP_SANITY_ASSERT(__LINE__); False positive for migrate-fail-7 */
2425         }
2426 
2427         g_list_free(possible_matches);
2428         return;
2429     }
2430 
2431     if(stop
2432        && (rsc->next_role == RSC_ROLE_STOPPED
2433            || (start && is_not_set(start->flags, pe_action_runnable)))) {
2434 
2435         GListPtr gIter = NULL;
2436 
2437         key = stop_key(rsc);
2438         for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
2439             node_t *node = (node_t *) gIter->data;
2440             action_t *stop_op = NULL;
2441 
2442             possible_matches = find_actions(rsc->actions, key, node);
2443             if (possible_matches) {
2444                 stop_op = possible_matches->data;
2445                 g_list_free(possible_matches);
2446             }
2447 
2448             if (stop_op && (stop_op->flags & pe_action_runnable)) {
2449                 STOP_SANITY_ASSERT(__LINE__);
2450             }
2451 
2452             LogAction("Stop", rsc, node, NULL, stop_op,
2453                       (stop_op && stop_op->reason)? stop_op : start, terminal);
2454         }
2455 
2456         free(key);
2457 
2458     } else if (stop && is_set(rsc->flags, pe_rsc_failed)) {
2459         /* 'stop' may be NULL if the failure was ignored */
2460         LogAction("Recover", rsc, current, next, stop, start, terminal);
2461         STOP_SANITY_ASSERT(__LINE__);
2462 
2463     } else if (moving) {
2464         LogAction("Move", rsc, current, next, stop, NULL, terminal);
2465         STOP_SANITY_ASSERT(__LINE__);
2466 
2467     } else if (is_set(rsc->flags, pe_rsc_reload)) {
2468         LogAction("Reload", rsc, current, next, start, NULL, terminal);
2469 
2470     } else if (stop != NULL && is_not_set(stop->flags, pe_action_optional)) {
2471         LogAction("Restart", rsc, current, next, start, NULL, terminal);
2472         STOP_SANITY_ASSERT(__LINE__);
2473 
2474     } else if (rsc->role == RSC_ROLE_MASTER) {
2475         CRM_LOG_ASSERT(current != NULL);
2476         LogAction("Demote", rsc, current, next, demote, NULL, terminal);
2477 
2478     } else if(rsc->next_role == RSC_ROLE_MASTER) {
2479         CRM_LOG_ASSERT(next);
2480         LogAction("Promote", rsc, current, next, promote, NULL, terminal);
2481 
2482     } else if (rsc->role == RSC_ROLE_STOPPED && rsc->next_role > RSC_ROLE_STOPPED) {
2483         LogAction("Start", rsc, current, next, start, NULL, terminal);
2484     }
2485 }
2486 
2487 gboolean
2488 StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
     /*  */
2489 {
2490     GListPtr gIter = NULL;
2491 
2492     CRM_ASSERT(rsc);
2493     pe_rsc_trace(rsc, "%s", rsc->id);
2494 
2495     for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
2496         node_t *current = (node_t *) gIter->data;
2497         action_t *stop;
2498 
2499         if (rsc->partial_migration_target) {
2500             if (rsc->partial_migration_target->details == current->details) {
2501                 pe_rsc_trace(rsc, "Filtered %s -> %s %s", current->details->uname,
2502                              next->details->uname, rsc->id);
2503                 continue;
2504             } else {
2505                 pe_rsc_trace(rsc, "Forced on %s %s", current->details->uname, rsc->id);
2506                 optional = FALSE;
2507             }
2508         }
2509 
2510         pe_rsc_trace(rsc, "%s on %s", rsc->id, current->details->uname);
2511         stop = stop_action(rsc, current, optional);
2512 
2513         if(rsc->allocated_to == NULL) {
2514             pe_action_set_reason(stop, "node availability", TRUE);
2515         }
2516 
2517         if (is_not_set(rsc->flags, pe_rsc_managed)) {
2518             update_action_flags(stop, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
2519         }
2520 
2521         if (is_set(data_set->flags, pe_flag_remove_after_stop)) {
2522             DeleteRsc(rsc, current, optional, data_set);
2523         }
2524 
2525         if(is_set(rsc->flags, pe_rsc_needs_unfencing)) {
2526             action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set);
2527             const char *unfenced = pe_node_attribute_raw(current, CRM_ATTR_UNFENCED);
2528 
2529             order_actions(stop, unfence, pe_order_implies_first);
2530             if (unfenced == NULL || safe_str_eq("0", unfenced)) {
2531                 pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname);
2532             }
2533         }
2534     }
2535 
2536     return TRUE;
2537 }
2538 
2539 gboolean
2540 StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
     /*  */
2541 {
2542     action_t *start = NULL;
2543 
2544     CRM_ASSERT(rsc);
2545     pe_rsc_trace(rsc, "%s on %s %d %d", rsc->id, next ? next->details->uname : "N/A", optional, next ? next->weight : 0);
2546     start = start_action(rsc, next, TRUE);
2547 
2548     if(is_set(rsc->flags, pe_rsc_needs_unfencing)) {
2549         action_t *unfence = pe_fence_op(next, "on", TRUE, NULL, data_set);
2550         const char *unfenced = pe_node_attribute_raw(next, CRM_ATTR_UNFENCED);
2551 
2552         order_actions(unfence, start, pe_order_implies_then);
2553 
2554         if (unfenced == NULL || safe_str_eq("0", unfenced)) {
2555             char *reason = crm_strdup_printf("Required by %s", rsc->id);
2556             trigger_unfencing(NULL, next, reason, NULL, data_set);
2557             free(reason);
2558         }
2559     }
2560 
2561     if (is_set(start->flags, pe_action_runnable) && optional == FALSE) {
2562         update_action_flags(start, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
2563     }
2564 
2565 
2566     return TRUE;
2567 }
2568 
2569 gboolean
2570 PromoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
     /*  */
2571 {
2572     char *key = NULL;
2573     GListPtr gIter = NULL;
2574     gboolean runnable = TRUE;
2575     GListPtr action_list = NULL;
2576 
2577     CRM_ASSERT(rsc);
2578     CRM_CHECK(next != NULL, return FALSE);
2579     pe_rsc_trace(rsc, "%s on %s", rsc->id, next->details->uname);
2580 
2581     key = start_key(rsc);
2582     action_list = find_actions_exact(rsc->actions, key, next);
2583     free(key);
2584 
2585     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
2586         action_t *start = (action_t *) gIter->data;
2587 
2588         if (is_set(start->flags, pe_action_runnable) == FALSE) {
2589             runnable = FALSE;
2590         }
2591     }
2592     g_list_free(action_list);
2593 
2594     if (runnable) {
2595         promote_action(rsc, next, optional);
2596         return TRUE;
2597     }
2598 
2599     pe_rsc_debug(rsc, "%s\tPromote %s (canceled)", next->details->uname, rsc->id);
2600 
2601     key = promote_key(rsc);
2602     action_list = find_actions_exact(rsc->actions, key, next);
2603     free(key);
2604 
2605     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
2606         action_t *promote = (action_t *) gIter->data;
2607 
2608         update_action_flags(promote, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
2609     }
2610 
2611     g_list_free(action_list);
2612     return TRUE;
2613 }
2614 
2615 gboolean
2616 DemoteRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
     /*  */
2617 {
2618     GListPtr gIter = NULL;
2619 
2620     CRM_ASSERT(rsc);
2621     pe_rsc_trace(rsc, "%s", rsc->id);
2622 
2623 /*      CRM_CHECK(rsc->next_role == RSC_ROLE_SLAVE, return FALSE); */
2624     for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) {
2625         node_t *current = (node_t *) gIter->data;
2626 
2627         pe_rsc_trace(rsc, "%s on %s", rsc->id, next ? next->details->uname : "N/A");
2628         demote_action(rsc, current, optional);
2629     }
2630     return TRUE;
2631 }
2632 
2633 gboolean
2634 RoleError(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
     /*  */
2635 {
2636     CRM_ASSERT(rsc);
2637     crm_err("%s on %s", rsc->id, next ? next->details->uname : "N/A");
2638     CRM_CHECK(FALSE, return FALSE);
2639     return FALSE;
2640 }
2641 
2642 gboolean
2643 NullOp(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * data_set)
     /*  */
2644 {
2645     CRM_ASSERT(rsc);
2646     pe_rsc_trace(rsc, "%s", rsc->id);
2647     return FALSE;
2648 }
2649 
2650 gboolean
2651 DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set)
     /*  */
2652 {
2653     if (is_set(rsc->flags, pe_rsc_failed)) {
2654         pe_rsc_trace(rsc, "Resource %s not deleted from %s: failed", rsc->id, node->details->uname);
2655         return FALSE;
2656 
2657     } else if (node == NULL) {
2658         pe_rsc_trace(rsc, "Resource %s not deleted: NULL node", rsc->id);
2659         return FALSE;
2660 
2661     } else if (node->details->unclean || node->details->online == FALSE) {
2662         pe_rsc_trace(rsc, "Resource %s not deleted from %s: unrunnable", rsc->id,
2663                      node->details->uname);
2664         return FALSE;
2665     }
2666 
2667     crm_notice("Removing %s from %s", rsc->id, node->details->uname);
2668 
2669     delete_action(rsc, node, optional);
2670 
2671     new_rsc_order(rsc, RSC_STOP, rsc, RSC_DELETE,
2672                   optional ? pe_order_implies_then : pe_order_optional, data_set);
2673 
2674     new_rsc_order(rsc, RSC_DELETE, rsc, RSC_START,
2675                   optional ? pe_order_implies_then : pe_order_optional, data_set);
2676 
2677     return TRUE;
2678 }
2679 
2680 #include <../lib/pengine/unpack.h>
2681 #define set_char(x) last_rsc_id[lpc] = x; complete = TRUE;
2682 static char *
2683 increment_clone(char *last_rsc_id)
     /*  */
2684 {
2685     int lpc = 0;
2686     int len = 0;
2687     char *tmp = NULL;
2688     gboolean complete = FALSE;
2689 
2690     CRM_CHECK(last_rsc_id != NULL, return NULL);
2691     if (last_rsc_id != NULL) {
2692         len = strlen(last_rsc_id);
2693     }
2694 
2695     lpc = len - 1;
2696     while (complete == FALSE && lpc > 0) {
2697         switch (last_rsc_id[lpc]) {
2698             case 0:
2699                 lpc--;
2700                 break;
2701             case '0':
2702                 set_char('1');
2703                 break;
2704             case '1':
2705                 set_char('2');
2706                 break;
2707             case '2':
2708                 set_char('3');
2709                 break;
2710             case '3':
2711                 set_char('4');
2712                 break;
2713             case '4':
2714                 set_char('5');
2715                 break;
2716             case '5':
2717                 set_char('6');
2718                 break;
2719             case '6':
2720                 set_char('7');
2721                 break;
2722             case '7':
2723                 set_char('8');
2724                 break;
2725             case '8':
2726                 set_char('9');
2727                 break;
2728             case '9':
2729                 last_rsc_id[lpc] = '0';
2730                 lpc--;
2731                 break;
2732             case ':':
2733                 tmp = last_rsc_id;
2734                 last_rsc_id = calloc(1, len + 2);
2735                 memcpy(last_rsc_id, tmp, len);
2736                 last_rsc_id[++lpc] = '1';
2737                 last_rsc_id[len] = '0';
2738                 last_rsc_id[len + 1] = 0;
2739                 complete = TRUE;
2740                 free(tmp);
2741                 break;
2742             default:
2743                 crm_err("Unexpected char: %c (%d)", last_rsc_id[lpc], lpc);
2744                 return NULL;
2745                 break;
2746         }
2747     }
2748     return last_rsc_id;
2749 }
2750 
2751 static node_t *
2752 probe_anon_group_member(resource_t *rsc, node_t *node,
     /*  */
2753                         pe_working_set_t *data_set)
2754 {
2755     resource_t *top = uber_parent(rsc);
2756 
2757     if (is_not_set(top->flags, pe_rsc_unique)) {
2758         /* Annoyingly we also need to check any other clone instances
2759          * Clumsy, but it will work.
2760          *
2761          * An alternative would be to update known_on for every peer
2762          * during process_rsc_state()
2763          *
2764          * This code desperately needs optimization
2765          * ptest -x with 100 nodes, 100 clones and clone-max=10:
2766          *   No probes                          O(25s)
2767          *   Detection without clone loop               O(3m)
2768          *   Detection with clone loop                  O(8m)
2769 
2770          ptest[32211]: 2010/02/18_14:27:55 CRIT: stage5: Probing for unknown resources
2771          ptest[32211]: 2010/02/18_14:33:39 CRIT: stage5: Done
2772          ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Updating action states
2773          ptest[32211]: 2010/02/18_14:35:05 CRIT: stage7: Done
2774 
2775          */
2776         char *clone_id = clone_zero(rsc->id);
2777         resource_t *peer = pe_find_resource(top->children, clone_id);
2778         node_t *running = NULL;
2779 
2780         while (peer) {
2781             running = pe_hash_table_lookup(peer->known_on, node->details->id);
2782             if (running != NULL) {
2783                 /* we already know the status of the resource on this node */
2784                 pe_rsc_trace(rsc, "Skipping active clone: %s", rsc->id);
2785                 free(clone_id);
2786                 return running;
2787             }
2788             clone_id = increment_clone(clone_id);
2789             peer = pe_find_resource(data_set->resources, clone_id);
2790         }
2791 
2792         free(clone_id);
2793     }
2794     return NULL;
2795 }
2796 
2797 gboolean
2798 native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
     /*  */
2799                     gboolean force, pe_working_set_t * data_set)
2800 {
2801     enum pe_ordering flags = pe_order_optional;
2802     char *key = NULL;
2803     action_t *probe = NULL;
2804     node_t *running = NULL;
2805     node_t *allowed = NULL;
2806     resource_t *top = uber_parent(rsc);
2807 
2808     static const char *rc_master = NULL;
2809     static const char *rc_inactive = NULL;
2810 
2811     if (rc_inactive == NULL) {
2812         rc_inactive = crm_itoa(PCMK_OCF_NOT_RUNNING);
2813         rc_master = crm_itoa(PCMK_OCF_RUNNING_MASTER);
2814     }
2815 
2816     CRM_CHECK(node != NULL, return FALSE);
2817     if (force == FALSE && is_not_set(data_set->flags, pe_flag_startup_probes)) {
2818         pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id);
2819         return FALSE;
2820     }
2821 
2822     if (is_remote_node(node)) {
2823         const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
2824 
2825         if (safe_str_eq(class, PCMK_RESOURCE_CLASS_STONITH)) {
2826             pe_rsc_trace(rsc,
2827                          "Skipping probe for %s on %s because Pacemaker Remote nodes cannot run stonith agents",
2828                          rsc->id, node->details->id);
2829             return FALSE;
2830         } else if (is_container_remote_node(node) && rsc_contains_remote_node(data_set, rsc)) {
2831             pe_rsc_trace(rsc,
2832                          "Skipping probe for %s on %s because guest nodes cannot run resources containing guest nodes",
2833                          rsc->id, node->details->id);
2834             return FALSE;
2835         } else if (rsc->is_remote_node) {
2836             pe_rsc_trace(rsc,
2837                          "Skipping probe for %s on %s because Pacemaker Remote nodes cannot host remote connections",
2838                          rsc->id, node->details->id);
2839             return FALSE;
2840         }
2841     }
2842 
2843     if (rsc->children) {
2844         GListPtr gIter = NULL;
2845         gboolean any_created = FALSE;
2846 
2847         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
2848             resource_t *child_rsc = (resource_t *) gIter->data;
2849 
2850             any_created = child_rsc->cmds->create_probe(child_rsc, node, complete, force, data_set)
2851                 || any_created;
2852         }
2853 
2854         return any_created;
2855 
2856     } else if ((rsc->container) && (!rsc->is_remote_node)) {
2857         pe_rsc_trace(rsc, "Skipping %s: it is within container %s", rsc->id, rsc->container->id);
2858         return FALSE;
2859     }
2860 
2861     if (is_set(rsc->flags, pe_rsc_orphan)) {
2862         pe_rsc_trace(rsc, "Skipping orphan: %s", rsc->id);
2863         return FALSE;
2864     }
2865 
2866     running = g_hash_table_lookup(rsc->known_on, node->details->id);
2867     if (running == NULL && is_set(rsc->flags, pe_rsc_unique) == FALSE) {
2868         /* Anonymous clones */
2869         if (rsc->parent == top) {
2870             running = g_hash_table_lookup(rsc->parent->known_on, node->details->id);
2871 
2872         } else {
2873             // Members of anonymous-cloned groups need special handling
2874             running = probe_anon_group_member(rsc, node, data_set);
2875         }
2876     }
2877 
2878     if (force == FALSE && running != NULL) {
2879         /* we already know the status of the resource on this node */
2880         pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);
2881         return FALSE;
2882     }
2883 
2884     allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
2885 
2886     if (rsc->exclusive_discover || top->exclusive_discover) {
2887         if (allowed == NULL) {
2888             /* exclusive discover is enabled and this node is not in the allowed list. */    
2889             pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id);
2890             return FALSE;
2891         } else if (allowed->rsc_discover_mode != pe_discover_exclusive) {
2892             /* exclusive discover is enabled and this node is not marked
2893              * as a node this resource should be discovered on */ 
2894             pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id);
2895             return FALSE;
2896         }
2897     }
2898 
2899     if(allowed == NULL && node->rsc_discover_mode == pe_discover_never) {
2900         /* If this node was allowed to host this resource it would
2901          * have been explicitly added to the 'allowed_nodes' list.
2902          * However it wasn't and the node has discovery disabled, so
2903          * no need to probe for this resource.
2904          */
2905         pe_rsc_trace(rsc, "Skipping probe for %s on node %s, C", rsc->id, node->details->id);
2906         return FALSE;
2907     }
2908 
2909     if (allowed && allowed->rsc_discover_mode == pe_discover_never) {
2910         /* this resource is marked as not needing to be discovered on this node */
2911         pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id);
2912         return FALSE;
2913     }
2914 
2915     if(is_container_remote_node(node)) {
2916         resource_t *remote = node->details->remote_rsc->container;
2917 
2918         if(remote->role == RSC_ROLE_STOPPED) {
2919             /* If the container is stopped, then we know anything that
2920              * might have been inside it is also stopped and there is
2921              * no need to probe.
2922              *
2923              * If we don't know the container's state on the target
2924              * either:
2925              *
2926              * - the container is running, the transition will abort
2927              *   and we'll end up in a different case next time, or
2928              *
2929              * - the container is stopped
2930              *
2931              * Either way there is no need to probe.
2932              *
2933              */
2934             if(remote->allocated_to
2935                && g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) {
2936                 /* For safety, we order the 'rsc' start after 'remote'
2937                  * has been probed.
2938                  *
2939                  * Using 'top' helps for groups, but in we may need to
2940                  * follow the start's ordering chain backwards.
2941                  */
2942                 custom_action_order(remote, generate_op_key(remote->id, RSC_STATUS, 0), NULL,
2943                                     top, generate_op_key(top->id, RSC_START, 0), NULL,
2944                                     pe_order_optional, data_set);
2945             }
2946             pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopped",
2947                          rsc->id, node->details->id, remote->id);
2948             return FALSE;
2949 
2950             /* Here we really we want to check if remote->stop is required,
2951              * but that information doesn't exist yet
2952              */
2953         } else if(node->details->remote_requires_reset
2954                   || node->details->unclean
2955                   || is_set(remote->flags, pe_rsc_failed)
2956                   || remote->next_role == RSC_ROLE_STOPPED
2957                   || (remote->allocated_to
2958                       && pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL)
2959             ) {
2960             /* The container is stopping or restarting, don't start
2961              * 'rsc' until 'remote' stops as this also implies that
2962              * 'rsc' is stopped - avoiding the need to probe
2963              */
2964             custom_action_order(remote, generate_op_key(remote->id, RSC_STOP, 0), NULL,
2965                                 top, generate_op_key(top->id, RSC_START, 0), NULL,
2966                                 pe_order_optional, data_set);
2967         pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopping, restarting or moving",
2968                      rsc->id, node->details->id, remote->id);
2969             return FALSE;
2970 /*      } else {
2971  *            The container is running so there is no problem probing it
2972  */
2973         }
2974     }
2975 
2976     key = generate_op_key(rsc->id, RSC_STATUS, 0);
2977     probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);
2978     update_action_flags(probe, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__);
2979 
2980     /* If enabled, require unfencing before probing any fence devices
2981      * but ensure it happens after any resources that require
2982      * unfencing have been probed.
2983      *
2984      * Doing it the other way (requiring unfencing after probing
2985      * resources that need it) would result in the node being
2986      * unfenced, and all its resources being stopped, whenever a new
2987      * resource is added.  Which would be highly suboptimal.
2988      *
2989      * So essentially, at the point the fencing device(s) have been
2990      * probed, we know the state of all resources that require
2991      * unfencing and that unfencing occurred.
2992      */
2993     if(is_set(rsc->flags, pe_rsc_needs_unfencing)) {
2994         action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set);
2995         order_actions(unfence, probe, pe_order_optional);
2996     }
2997 
2998     /*
2999      * We need to know if it's running_on (not just known_on) this node
3000      * to correctly determine the target rc.
3001      */
3002     running = pe_find_node_id(rsc->running_on, node->details->id);
3003     if (running == NULL) {
3004         add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_inactive);
3005 
3006     } else if (rsc->role == RSC_ROLE_MASTER) {
3007         add_hash_param(probe->meta, XML_ATTR_TE_TARGET_RC, rc_master);
3008     }
3009 
3010     crm_debug("Probing %s on %s (%s) %d %p", rsc->id, node->details->uname, role2text(rsc->role),
3011               is_set(probe->flags, pe_action_runnable), rsc->running_on);
3012 
3013     if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) {
3014         top = rsc;
3015 
3016     } else if (pe_rsc_is_clone(top) == FALSE) {
3017         top = rsc;
3018 
3019     } else {
3020         crm_trace("Probing %s on %s (%s) as %s", rsc->id, node->details->uname, role2text(rsc->role), top->id);
3021     }
3022 
3023     if(is_not_set(probe->flags, pe_action_runnable) && rsc->running_on == NULL) {
3024         /* Prevent the start from occurring if rsc isn't active, but
3025          * don't cause it to stop if it was active already
3026          */
3027         flags |= pe_order_runnable_left;
3028     }
3029 
3030     custom_action_order(rsc, NULL, probe,
3031                         top, generate_op_key(top->id, RSC_START, 0), NULL,
3032                         flags, data_set);
3033 
3034     /* Before any reloads, if they exist */
3035     custom_action_order(rsc, NULL, probe,
3036                         top, reload_key(rsc), NULL,
3037                         pe_order_optional, data_set);
3038 
3039     if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) {
3040         /* Normally rsc.start depends on probe complete which depends
3041          * on rsc.probe. But this can't be the case in this scenario as
3042          * it would create graph loops.
3043          *
3044          * So instead we explicitly order 'rsc.probe then rsc.start'
3045          */
3046 
3047     } else {
3048         order_actions(probe, complete, pe_order_implies_then);
3049     }
3050     return TRUE;
3051 }
3052 
3053 static void
3054 native_start_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
     /*  */
3055 {
3056     node_t *target;
3057     GListPtr gIter = NULL;
3058     action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
3059     action_t *stonith_done = get_pseudo_op(STONITH_DONE, data_set);
3060 
3061     CRM_CHECK(stonith_op && stonith_op->node, return);
3062     target = stonith_op->node;
3063 
3064     for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
3065         action_t *action = (action_t *) gIter->data;
3066 
3067         if(action->needs == rsc_req_nothing) {
3068             /* Anything other than start or promote requires nothing */
3069 
3070         } else if (action->needs == rsc_req_stonith) {
3071             order_actions(stonith_done, action, pe_order_optional);
3072 
3073         } else if (safe_str_eq(action->task, RSC_START)
3074                    && NULL == pe_hash_table_lookup(rsc->known_on, target->details->id)) {
3075             /* if known == NULL, then we don't know if
3076              *   the resource is active on the node
3077              *   we're about to shoot
3078              *
3079              * in this case, regardless of action->needs,
3080              *   the only safe option is to wait until
3081              *   the node is shot before doing anything
3082              *   to with the resource
3083              *
3084              * it's analogous to waiting for all the probes
3085              *   for rscX to complete before starting rscX
3086              *
3087              * the most likely explanation is that the
3088              *   DC died and took its status with it
3089              */
3090 
3091             pe_rsc_debug(rsc, "Ordering %s after %s recovery", action->uuid,
3092                          target->details->uname);
3093             order_actions(all_stopped, action, pe_order_optional | pe_order_runnable_left);
3094         }
3095     }
3096 }
3097 
3098 static void
3099 native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
     /*  */
3100 {
3101     char *key = NULL;
3102     GListPtr gIter = NULL;
3103     GListPtr action_list = NULL;
3104 
3105     action_t *start = NULL;
3106     resource_t *top = uber_parent(rsc);
3107     node_t *target;
3108 
3109     CRM_CHECK(stonith_op && stonith_op->node, return);
3110     target = stonith_op->node;
3111 
3112     /* Check whether the resource has a pending start action */
3113     start = find_first_action(rsc->actions, NULL, CRMD_ACTION_START, NULL);
3114 
3115     /* Get a list of stop actions potentially implied by the fencing */
3116     key = stop_key(rsc);
3117     action_list = find_actions(rsc->actions, key, target);
3118     free(key);
3119 
3120     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
3121         action_t *action = (action_t *) gIter->data;
3122 
3123         if (is_set(rsc->flags, pe_rsc_failed)) {
3124             crm_notice("Stop of failed resource %s is implicit after %s is fenced",
3125                        rsc->id, target->details->uname);
3126         } else {
3127             crm_info("%s is implicit after %s is fenced",
3128                      action->uuid, target->details->uname);
3129         }
3130 
3131         /* The stop would never complete and is now implied by the fencing,
3132          * so convert it into a pseudo-action.
3133          */
3134         update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__);
3135         update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__);
3136         update_action_flags(action, pe_action_implied_by_stonith, __FUNCTION__, __LINE__);
3137 
3138         if(start == NULL || start->needs > rsc_req_quorum) {
3139             enum pe_ordering flags = pe_order_optional;
3140             action_t *parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
3141 
3142             if (target->details->remote_rsc) {
3143                 /* User constraints must not order a resource in a guest node
3144                  * relative to the guest node container resource. This flag
3145                  * marks constraints as generated by the cluster and thus
3146                  * immune to that check.
3147                  */
3148                 flags |= pe_order_preserve;
3149             }
3150             order_actions(stonith_op, action, flags);
3151             order_actions(stonith_op, parent_stop, flags);
3152         }
3153 
3154         if (is_set(rsc->flags, pe_rsc_notify)) {
3155             /* Create a second notification that will be delivered
3156              *   immediately after the node is fenced
3157              *
3158              * Basic problem:
3159              * - C is a clone active on the node to be shot and stopping on another
3160              * - R is a resource that depends on C
3161              *
3162              * + C.stop depends on R.stop
3163              * + C.stopped depends on STONITH
3164              * + C.notify depends on C.stopped
3165              * + C.healthy depends on C.notify
3166              * + R.stop depends on C.healthy
3167              *
3168              * The extra notification here changes
3169              *  + C.healthy depends on C.notify
3170              * into:
3171              *  + C.healthy depends on C.notify'
3172              *  + C.notify' depends on STONITH'
3173              * thus breaking the loop
3174              */
3175             create_secondary_notification(action, rsc, stonith_op, data_set);
3176         }
3177 
3178 /* From Bug #1601, successful fencing must be an input to a failed resources stop action.
3179 
3180    However given group(rA, rB) running on nodeX and B.stop has failed,
3181    A := stop healthy resource (rA.stop)
3182    B := stop failed resource (pseudo operation B.stop)
3183    C := stonith nodeX
3184    A requires B, B requires C, C requires A
3185    This loop would prevent the cluster from making progress.
3186 
3187    This block creates the "C requires A" dependency and therefore must (at least
3188    for now) be disabled.
3189 
3190    Instead, run the block above and treat all resources on nodeX as B would be
3191    (marked as a pseudo op depending on the STONITH).
3192 
3193    TODO: Break the "A requires B" dependency in update_action() and re-enable this block
3194 
3195    } else if(is_stonith == FALSE) {
3196    crm_info("Moving healthy resource %s"
3197    " off %s before fencing",
3198    rsc->id, node->details->uname);
3199 
3200    * stop healthy resources before the
3201    * stonith op
3202    *
3203    custom_action_order(
3204    rsc, stop_key(rsc), NULL,
3205    NULL,strdup(CRM_OP_FENCE),stonith_op,
3206    pe_order_optional, data_set);
3207 */
3208     }
3209 
3210     g_list_free(action_list);
3211 
3212     /* Get a list of demote actions potentially implied by the fencing */
3213     key = demote_key(rsc);
3214     action_list = find_actions(rsc->actions, key, target);
3215     free(key);
3216 
3217     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
3218         action_t *action = (action_t *) gIter->data;
3219 
3220         if (action->node->details->online == FALSE || action->node->details->unclean == TRUE
3221             || is_set(rsc->flags, pe_rsc_failed)) {
3222 
3223             if (is_set(rsc->flags, pe_rsc_failed)) {
3224                 pe_rsc_info(rsc,
3225                             "Demote of failed resource %s is implicit after %s is fenced",
3226                             rsc->id, target->details->uname);
3227             } else {
3228                 pe_rsc_info(rsc, "%s is implicit after %s is fenced",
3229                             action->uuid, target->details->uname);
3230             }
3231 
3232             /* The demote would never complete and is now implied by the
3233              * fencing, so convert it into a pseudo-action.
3234              */
3235             update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__);
3236             update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__);
3237 
3238             if (start == NULL || start->needs > rsc_req_quorum) {
3239                 order_actions(stonith_op, action, pe_order_preserve|pe_order_optional);
3240             }
3241         }
3242     }
3243 
3244     g_list_free(action_list);
3245 }
3246 
3247 void
3248 rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
     /*  */
3249 {
3250     if (rsc->children) {
3251         GListPtr gIter = NULL;
3252 
3253         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
3254             resource_t *child_rsc = (resource_t *) gIter->data;
3255 
3256             rsc_stonith_ordering(child_rsc, stonith_op, data_set);
3257         }
3258 
3259     } else if (is_not_set(rsc->flags, pe_rsc_managed)) {
3260         pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id);
3261 
3262     } else {
3263         native_start_constraints(rsc, stonith_op, data_set);
3264         native_stop_constraints(rsc, stonith_op, data_set);
3265     }
3266 }
3267 
3268 enum stack_activity {
3269     stack_stable = 0,
3270     stack_starting = 1,
3271     stack_stopping = 2,
3272     stack_middle = 4,
3273 };
3274 
3275 void
3276 ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set)
     /*  */
3277 {
3278     GListPtr gIter = NULL;
3279     action_t *reload = NULL;
3280 
3281     if (rsc->children) {
3282         for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
3283             resource_t *child_rsc = (resource_t *) gIter->data;
3284 
3285             ReloadRsc(child_rsc, node, data_set);
3286         }
3287         return;
3288 
3289     } else if (rsc->variant > pe_native) {
3290         /* Complex resource with no children */
3291         return;
3292 
3293     } else if (is_not_set(rsc->flags, pe_rsc_managed)) {
3294         pe_rsc_trace(rsc, "%s: unmanaged", rsc->id);
3295         return;
3296 
3297     } else if (is_set(rsc->flags, pe_rsc_failed) || is_set(rsc->flags, pe_rsc_start_pending)) {
3298         pe_rsc_trace(rsc, "%s: general resource state: flags=0x%.16llx", rsc->id, rsc->flags);
3299         stop_action(rsc, node, FALSE); /* Force a full restart, overkill? */
3300         return;
3301 
3302     } else if (node == NULL) {
3303         pe_rsc_trace(rsc, "%s: not active", rsc->id);
3304         return;
3305     }
3306 
3307     pe_rsc_trace(rsc, "Processing %s", rsc->id);
3308     set_bit(rsc->flags, pe_rsc_reload);
3309 
3310     reload = custom_action(
3311         rsc, reload_key(rsc), CRMD_ACTION_RELOAD, node, FALSE, TRUE, data_set);
3312     pe_action_set_reason(reload, "resource definition change", FALSE);
3313 
3314     custom_action_order(NULL, NULL, reload, rsc, stop_key(rsc), NULL,
3315                         pe_order_optional|pe_order_then_cancels_first,
3316                         data_set);
3317     custom_action_order(NULL, NULL, reload, rsc, demote_key(rsc), NULL,
3318                         pe_order_optional|pe_order_then_cancels_first,
3319                         data_set);
3320 }
3321 
3322 void
3323 native_append_meta(resource_t * rsc, xmlNode * xml)
     /*  */
3324 {
3325     char *value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
3326     resource_t *iso_parent, *last_parent, *parent;
3327 
3328     if (value) {
3329         char *name = NULL;
3330 
3331         name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
3332         crm_xml_add(xml, name, value);
3333         free(name);
3334     }
3335 
3336     value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
3337     if (value) {
3338         char *name = NULL;
3339 
3340         name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
3341         crm_xml_add(xml, name, value);
3342         free(name);
3343     }
3344 
3345     for (parent = rsc; parent != NULL; parent = parent->parent) {
3346         if (parent->container) {
3347             crm_xml_add(xml, CRM_META"_"XML_RSC_ATTR_CONTAINER, parent->container->id);
3348         }
3349     }
3350 
3351     last_parent = iso_parent = rsc;
3352     while (iso_parent != NULL) {
3353         char *name = NULL;
3354         char *iso = NULL;
3355 
3356         if (iso_parent->isolation_wrapper == NULL) {
3357             last_parent = iso_parent;
3358             iso_parent = iso_parent->parent;
3359             continue;
3360         }
3361 
3362         /* name of wrapper script this resource is routed through. */
3363         name = crm_meta_name(XML_RSC_ATTR_ISOLATION_WRAPPER);
3364         crm_xml_add(xml, name, iso_parent->isolation_wrapper);
3365         free(name);
3366 
3367         /* instance name for isolated environment */
3368         name = crm_meta_name(XML_RSC_ATTR_ISOLATION_INSTANCE);
3369         if (pe_rsc_is_clone(iso_parent)) { 
3370             /* if isolation is set at the clone/master level, we have to 
3371              * give this resource the unique isolation instance associated
3372              * with the clone child (last_parent)*/
3373 
3374             /* Example: cloned group. group is container
3375              * clone myclone - iso_parent
3376              *    group mygroup - last_parent (this is the iso environment)
3377              *       rsc myrsc1 - rsc
3378              *       rsc myrsc2
3379              * The group is what is isolated in example1. We have to make
3380              * sure myrsc1 and myrsc2 launch in the same isolated environment.
3381              *
3382              * Example: cloned primitives. rsc primitive is container
3383              * clone myclone iso_parent
3384              *     rsc myrsc1 - last_parent == rsc (this is the iso environment)
3385              * The individual cloned primitive instances are isolated
3386              */
3387             value = g_hash_table_lookup(last_parent->meta, XML_RSC_ATTR_INCARNATION);
3388             CRM_ASSERT(value != NULL);
3389 
3390             iso = crm_concat(crm_element_value(last_parent->xml, XML_ATTR_ID), value, '_');
3391             crm_xml_add(xml, name, iso);
3392             free(iso);
3393         } else { 
3394             /*
3395              * Example: cloned group of containers
3396              * clone myclone
3397              *    group mygroup
3398              *       rsc myrsc1 - iso_parent (this is the iso environment)
3399              *       rsc myrsc2
3400              *
3401              * Example: group of containers
3402              * group mygroup
3403              *   rsc myrsc1 - iso_parent (this is the iso environment)
3404              *   rsc myrsc2
3405              * 
3406              * Example: group is container
3407              * group mygroup - iso_parent ( this is iso environment)
3408              *   rsc myrsc1 
3409              *   rsc myrsc2
3410              *
3411              * Example: single primitive
3412              * rsc myrsc1 - iso_parent (this is the iso environment)
3413              */
3414             value = g_hash_table_lookup(iso_parent->meta, XML_RSC_ATTR_INCARNATION);
3415             if (value) {
3416                 crm_xml_add(xml, name, iso_parent->id);
3417                 iso = crm_concat(crm_element_value(iso_parent->xml, XML_ATTR_ID), value, '_');
3418                 crm_xml_add(xml, name, iso);
3419                 free(iso);
3420             } else {
3421                 crm_xml_add(xml, name, iso_parent->id);
3422             }
3423         }
3424         free(name);
3425 
3426         break;
3427     }
3428 }