root/crmd/te_actions.c

/* */

DEFINITIONS

1. te_start_action_timer
2. te_pseudo_action
3. send_stonith_update
4. te_fence_node
5. get_target_rc
6. te_crm_command
7. cib_action_update
8. te_rsc_command
9. te_peer_free
10. te_reset_job_counts
11. te_update_job_count_on
12. te_update_job_count
13. te_should_perform_action_on
14. te_should_perform_action
15. te_action_confirmed
16. notify_crmd

   1 /*
   2  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  *
   9  * This software is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 
  19 #include <crm_internal.h>
  20 
  21 #include <sys/param.h>
  22 #include <crm/crm.h>
  23 #include <crm/cib.h>
  24 #include <crm/msg_xml.h>
  25 
  26 #include <crm/common/xml.h>
  27 #include <tengine.h>
  28 
  29 #include <crmd_fsa.h>
  30 #include <crmd_lrm.h>
  31 #include <crmd_messages.h>
  32 #include <crm/cluster.h>
  33 #include <throttle.h>
  34 
  35 char *te_uuid = NULL;
  36 GHashTable *te_targets = NULL;
  37 void send_rsc_command(crm_action_t * action);
  38 static void te_update_job_count(crm_action_t * action, int offset);
  39 
  40 static void
  41 te_start_action_timer(crm_graph_t * graph, crm_action_t * action)
     /*  */
  42 {
  43     action->timer = calloc(1, sizeof(crm_action_timer_t));
  44     action->timer->timeout = action->timeout;
  45     action->timer->reason = timeout_action;
  46     action->timer->action = action;
  47     action->timer->source_id = g_timeout_add(action->timer->timeout + graph->network_delay,
  48                                              action_timer_callback, (void *)action->timer);
  49 
  50     CRM_ASSERT(action->timer->source_id != 0);
  51 }
  52 
  53 static gboolean
  54 te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
     /*  */
  55 {
  56     const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);
  57 
  58     /* send to peers as well? */
  59     if (safe_str_eq(task, CRM_OP_MAINTENANCE_NODES)) {
  60         GHashTableIter iter;
  61         crm_node_t *node = NULL;
  62 
  63         g_hash_table_iter_init(&iter, crm_peer_cache);
  64         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
  65             xmlNode *cmd = NULL;
  66 
  67             if (safe_str_eq(fsa_our_uname, node->uname)) {
  68                 continue;
  69             }
  70 
  71             cmd = create_request(task, pseudo->xml, node->uname,
  72                                  CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
  73             send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
  74             free_xml(cmd);
  75         }
  76 
  77         remote_ra_process_maintenance_nodes(pseudo->xml);
  78     } else {
  79         /* Check action for Pacemaker Remote node side effects */
  80         remote_ra_process_pseudo(pseudo->xml);
  81     }
  82 
  83     crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
  84               crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
  85     te_action_confirmed(pseudo);
  86     update_graph(graph, pseudo);
  87     trigger_graph();
  88     return TRUE;
  89 }
  90 
  91 void
  92 send_stonith_update(crm_action_t * action, const char *target, const char *uuid)
     /*  */
  93 {
  94     int rc = pcmk_ok;
  95     crm_node_t *peer = NULL;
  96 
  97     /* We (usually) rely on the membership layer to do node_update_cluster,
  98      * and the peer status callback to do node_update_peer, because the node
  99      * might have already rejoined before we get the stonith result here.
 100      */
 101     int flags = node_update_join | node_update_expected;
 102 
 103     /* zero out the node-status & remove all LRM status info */
 104     xmlNode *node_state = NULL;
 105 
 106     CRM_CHECK(target != NULL, return);
 107     CRM_CHECK(uuid != NULL, return);
 108 
 109     /* Make sure the membership and join caches are accurate */
 110     peer = crm_get_peer_full(0, target, CRM_GET_PEER_ANY);
 111 
 112     CRM_CHECK(peer != NULL, return);
 113 
 114     if (peer->state == NULL) {
 115         /* Usually, we rely on the membership layer to update the cluster state
 116          * in the CIB. However, if the node has never been seen, do it here, so
 117          * the node is not considered unclean.
 118          */
 119         flags |= node_update_cluster;
 120     }
 121 
 122     if (peer->uuid == NULL) {
 123         crm_info("Recording uuid '%s' for node '%s'", uuid, target);
 124         peer->uuid = strdup(uuid);
 125     }
 126 
 127     crmd_peer_down(peer, TRUE);
 128 
 129     /* Generate a node state update for the CIB */
 130     node_state = create_node_state_update(peer, flags, NULL, __FUNCTION__);
 131 
 132     /* we have to mark whether or not remote nodes have already been fenced */
 133     if (peer->flags & crm_remote_node) {
 134         time_t now = time(NULL);
 135         char *now_s = crm_itoa(now);
 136         crm_xml_add(node_state, XML_NODE_IS_FENCED, now_s);
 137         free(now_s);
 138     }
 139 
 140     /* Force our known ID */
 141     crm_xml_add(node_state, XML_ATTR_UUID, uuid);
 142 
 143     rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, node_state,
 144                                     cib_quorum_override | cib_scope_local | cib_can_create);
 145 
 146     /* Delay processing the trigger until the update completes */
 147     crm_debug("Sending fencing update %d for %s", rc, target);
 148     fsa_register_cib_callback(rc, FALSE, strdup(target), cib_fencing_updated);
 149 
 150     /* Make sure it sticks */
 151     /* fsa_cib_conn->cmds->bump_epoch(fsa_cib_conn, cib_quorum_override|cib_scope_local);    */
 152 
 153     erase_status_tag(peer->uname, XML_CIB_TAG_LRM, cib_scope_local);
 154     erase_status_tag(peer->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
 155 
 156     free_xml(node_state);
 157     return;
 158 }
 159 
 160 static gboolean
 161 te_fence_node(crm_graph_t * graph, crm_action_t * action)
     /*  */
 162 {
 163     int rc = 0;
 164     const char *id = NULL;
 165     const char *uuid = NULL;
 166     const char *target = NULL;
 167     const char *type = NULL;
 168     gboolean invalid_action = FALSE;
 169     enum stonith_call_options options = st_opt_none;
 170 
 171     id = ID(action->xml);
 172     target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 173     uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 174     type = crm_meta_value(action->params, "stonith_action");
 175 
 176     CRM_CHECK(id != NULL, invalid_action = TRUE);
 177     CRM_CHECK(uuid != NULL, invalid_action = TRUE);
 178     CRM_CHECK(type != NULL, invalid_action = TRUE);
 179     CRM_CHECK(target != NULL, invalid_action = TRUE);
 180 
 181     if (invalid_action) {
 182         crm_log_xml_warn(action->xml, "BadAction");
 183         return FALSE;
 184     }
 185 
 186     crm_notice("Requesting fencing (%s) of node %s "
 187                CRM_XS " action=%s timeout=%d",
 188                type, target, id, transition_graph->stonith_timeout);
 189 
 190     /* Passing NULL means block until we can connect... */
 191     te_connect_stonith(NULL);
 192 
 193     if (crmd_join_phase_count(crm_join_confirmed) == 1) {
 194         options |= st_opt_allow_suicide;
 195     }
 196 
 197     rc = stonith_api->cmds->fence(stonith_api, options, target, type,
 198                                   transition_graph->stonith_timeout / 1000, 0);
 199 
 200     stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000,
 201                                          st_opt_timeout_updates,
 202                                          generate_transition_key(transition_graph->id, action->id,
 203                                                                  0, te_uuid),
 204                                          "tengine_stonith_callback", tengine_stonith_callback);
 205 
 206     return TRUE;
 207 }
 208 
 209 static int
 210 get_target_rc(crm_action_t * action)
     /*  */
 211 {
 212     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 213 
 214     if (target_rc_s != NULL) {
 215         return crm_parse_int(target_rc_s, "0");
 216     }
 217     return 0;
 218 }
 219 
 220 static gboolean
 221 te_crm_command(crm_graph_t * graph, crm_action_t * action)
     /*  */
 222 {
 223     char *counter = NULL;
 224     xmlNode *cmd = NULL;
 225     gboolean is_local = FALSE;
 226 
 227     const char *id = NULL;
 228     const char *task = NULL;
 229     const char *value = NULL;
 230     const char *on_node = NULL;
 231     const char *router_node = NULL;
 232 
 233     gboolean rc = TRUE;
 234     gboolean no_wait = FALSE;
 235 
 236     id = ID(action->xml);
 237     task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 238     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 239     router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 240 
 241     if (!router_node) {
 242         router_node = on_node;
 243     }
 244 
 245     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
 246               crm_err("Corrupted command (id=%s) %s: no node", crm_str(id), crm_str(task));
 247               return FALSE);
 248 
 249     crm_info("Executing crm-event (%s): %s on %s%s%s",
 250              crm_str(id), crm_str(task), on_node,
 251              is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
 252 
 253     if (safe_str_eq(router_node, fsa_our_uname)) {
 254         is_local = TRUE;
 255     }
 256 
 257     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
 258     if (crm_is_true(value)) {
 259         no_wait = TRUE;
 260     }
 261 
 262     if (is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
 263         /* defer until everything else completes */
 264         crm_info("crm-event (%s) is a local shutdown", crm_str(id));
 265         graph->completion_action = tg_shutdown;
 266         graph->abort_reason = "local shutdown";
 267         te_action_confirmed(action);
 268         update_graph(graph, action);
 269         trigger_graph();
 270         return TRUE;
 271 
 272     } else if (safe_str_eq(task, CRM_OP_SHUTDOWN)) {
 273         crm_node_t *peer = crm_get_peer(0, router_node);
 274         crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN);
 275     }
 276 
 277     cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
 278 
 279     counter =
 280         generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
 281     crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
 282 
 283     rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_crmd, cmd, TRUE);
 284     free(counter);
 285     free_xml(cmd);
 286 
 287     if (rc == FALSE) {
 288         crm_err("Action %d failed: send", action->id);
 289         return FALSE;
 290 
 291     } else if (no_wait) {
 292         te_action_confirmed(action);
 293         update_graph(graph, action);
 294         trigger_graph();
 295 
 296     } else {
 297         if (action->timeout <= 0) {
 298             crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %dms instead",
 299                     action->id, task, on_node, action->timeout, graph->network_delay);
 300             action->timeout = graph->network_delay;
 301         }
 302         te_start_action_timer(graph, action);
 303     }
 304 
 305     return TRUE;
 306 }
 307 
 308 gboolean
 309 cib_action_update(crm_action_t * action, int status, int op_rc)
     /*  */
 310 {
 311     lrmd_event_data_t *op = NULL;
 312     xmlNode *state = NULL;
 313     xmlNode *rsc = NULL;
 314     xmlNode *xml_op = NULL;
 315     xmlNode *action_rsc = NULL;
 316 
 317     int rc = pcmk_ok;
 318 
 319     const char *rsc_id = NULL;
 320     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 321     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 322     const char *task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 323     const char *target_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
 324 
 325     int call_options = cib_quorum_override | cib_scope_local;
 326     int target_rc = get_target_rc(action);
 327 
 328     if (status == PCMK_LRM_OP_PENDING) {
 329         crm_debug("%s %d: Recording pending operation %s on %s",
 330                   crm_element_name(action->xml), action->id, task_uuid, target);
 331     } else {
 332         crm_warn("%s %d: %s on %s timed out",
 333                  crm_element_name(action->xml), action->id, task_uuid, target);
 334     }
 335 
 336     action_rsc = find_xml_node(action->xml, XML_CIB_TAG_RESOURCE, TRUE);
 337     if (action_rsc == NULL) {
 338         return FALSE;
 339     }
 340 
 341     rsc_id = ID(action_rsc);
 342     CRM_CHECK(rsc_id != NULL, crm_log_xml_err(action->xml, "Bad:action");
 343               return FALSE);
 344 
 345 /*
 346   update the CIB
 347 
 348 <node_state id="hadev">
 349       <lrm>
 350         <lrm_resources>
 351           <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
 352 */
 353 
 354     state = create_xml_node(NULL, XML_CIB_TAG_STATE);
 355 
 356     crm_xml_add(state, XML_ATTR_UUID, target_uuid);
 357     crm_xml_add(state, XML_ATTR_UNAME, target);
 358 
 359     rsc = create_xml_node(state, XML_CIB_TAG_LRM);
 360     crm_xml_add(rsc, XML_ATTR_ID, target_uuid);
 361 
 362     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCES);
 363     rsc = create_xml_node(rsc, XML_LRM_TAG_RESOURCE);
 364     crm_xml_add(rsc, XML_ATTR_ID, rsc_id);
 365 
 366 
 367     crm_copy_xml_element(action_rsc, rsc, XML_ATTR_TYPE);
 368     crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_CLASS);
 369     crm_copy_xml_element(action_rsc, rsc, XML_AGENT_ATTR_PROVIDER);
 370 
 371     op = convert_graph_action(NULL, action, status, op_rc);
 372     op->call_id = -1;
 373     op->user_data = generate_transition_key(transition_graph->id, action->id, target_rc, te_uuid);
 374 
 375     xml_op = create_operation_update(rsc, op, CRM_FEATURE_SET, target_rc, target, __FUNCTION__, LOG_INFO);
 376     lrmd_free_event(op);
 377 
 378     crm_trace("Updating CIB with \"%s\" (%s): %s %s on %s",
 379               status < 0 ? "new action" : XML_ATTR_TIMEOUT,
 380               crm_element_name(action->xml), crm_str(task), rsc_id, target);
 381     crm_log_xml_trace(xml_op, "Op");
 382 
 383     rc = fsa_cib_conn->cmds->update(fsa_cib_conn, XML_CIB_TAG_STATUS, state, call_options);
 384 
 385     crm_trace("Updating CIB with %s action %d: %s on %s (call_id=%d)",
 386               services_lrm_status_str(status), action->id, task_uuid, target, rc);
 387 
 388     fsa_register_cib_callback(rc, FALSE, NULL, cib_action_updated);
 389     free_xml(state);
 390 
 391     action->sent_update = TRUE;
 392 
 393     if (rc < pcmk_ok) {
 394         return FALSE;
 395     }
 396 
 397     return TRUE;
 398 }
 399 
 400 static gboolean
 401 te_rsc_command(crm_graph_t * graph, crm_action_t * action)
     /*  */
 402 {
 403     /* never overwrite stop actions in the CIB with
 404      *   anything other than completed results
 405      *
 406      * Writing pending stops makes it look like the
 407      *   resource is running again
 408      */
 409     xmlNode *cmd = NULL;
 410     xmlNode *rsc_op = NULL;
 411 
 412     gboolean rc = TRUE;
 413     gboolean no_wait = FALSE;
 414     gboolean is_local = FALSE;
 415 
 416     char *counter = NULL;
 417     const char *task = NULL;
 418     const char *value = NULL;
 419     const char *on_node = NULL;
 420     const char *router_node = NULL;
 421     const char *task_uuid = NULL;
 422 
 423     CRM_ASSERT(action != NULL);
 424     CRM_ASSERT(action->xml != NULL);
 425 
 426     action->executed = FALSE;
 427     on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 428 
 429     CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
 430               crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), crm_str(task));
 431               return FALSE);
 432 
 433     rsc_op = action->xml;
 434     task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 435     task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 436     router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);
 437 
 438     if (!router_node) {
 439         router_node = on_node;
 440     }
 441 
 442     counter =
 443         generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
 444     crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
 445 
 446     if (safe_str_eq(router_node, fsa_our_uname)) {
 447         is_local = TRUE;
 448     }
 449 
 450     value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
 451     if (crm_is_true(value)) {
 452         no_wait = TRUE;
 453     }
 454 
 455     crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
 456                task, task_uuid, (is_local? " locally" : ""), on_node,
 457                (no_wait? " without waiting" : ""), action->id);
 458 
 459     cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
 460                          CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
 461 
 462     if (is_local) {
 463         /* shortcut local resource commands */
 464         ha_msg_input_t data = {
 465             .msg = cmd,
 466             .xml = rsc_op,
 467         };
 468 
 469         fsa_data_t msg = {
 470             .id = 0,
 471             .data = &data,
 472             .data_type = fsa_dt_ha_msg,
 473             .fsa_input = I_NULL,
 474             .fsa_cause = C_FSA_INTERNAL,
 475             .actions = A_LRM_INVOKE,
 476             .origin = __FUNCTION__,
 477         };
 478 
 479         do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
 480 
 481     } else {
 482         rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
 483     }
 484 
 485     free(counter);
 486     free_xml(cmd);
 487 
 488     action->executed = TRUE;
 489 
 490     if (rc == FALSE) {
 491         crm_err("Action %d failed: send", action->id);
 492         return FALSE;
 493 
 494     } else if (no_wait) {
 495         crm_info("Action %d confirmed - no wait", action->id);
 496         action->confirmed = TRUE; /* Just mark confirmed.
 497                                    * Don't bump the job count only to immediately decrement it
 498                                    */
 499         update_graph(transition_graph, action);
 500         trigger_graph();
 501 
 502     } else if (action->confirmed == TRUE) {
 503         crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
 504                   action->id, task, task_uuid, on_node, action->timeout);
 505     } else {
 506         if (action->timeout <= 0) {
 507             crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %dms instead",
 508                     action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
 509             action->timeout = graph->network_delay;
 510         }
 511         te_update_job_count(action, 1);
 512         te_start_action_timer(graph, action);
 513     }
 514 
 515     return TRUE;
 516 }
 517 
 518 struct te_peer_s
 519 {
 520         char *name;
 521         int jobs;
 522         int migrate_jobs;
 523 };
 524 
 525 static void te_peer_free(gpointer p)
     /*  */
 526 {
 527     struct te_peer_s *peer = p;
 528 
 529     free(peer->name);
 530     free(peer);
 531 }
 532 
 533 void te_reset_job_counts(void)
     /*  */
 534 {
 535     GHashTableIter iter;
 536     struct te_peer_s *peer = NULL;
 537 
 538     if(te_targets == NULL) {
 539         te_targets = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, te_peer_free);
 540     }
 541 
 542     g_hash_table_iter_init(&iter, te_targets);
 543     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
 544         peer->jobs = 0;
 545         peer->migrate_jobs = 0;
 546     }
 547 }
 548 
 549 static void
 550 te_update_job_count_on(const char *target, int offset, bool migrate)
     /*  */
 551 {
 552     struct te_peer_s *r = NULL;
 553 
 554     if(target == NULL || te_targets == NULL) {
 555         return;
 556     }
 557 
 558     r = g_hash_table_lookup(te_targets, target);
 559     if(r == NULL) {
 560         r = calloc(1, sizeof(struct te_peer_s));
 561         r->name = strdup(target);
 562         g_hash_table_insert(te_targets, r->name, r);
 563     }
 564 
 565     r->jobs += offset;
 566     if(migrate) {
 567         r->migrate_jobs += offset;
 568     }
 569     crm_trace("jobs[%s] = %d", target, r->jobs);
 570 }
 571 
 572 static void
 573 te_update_job_count(crm_action_t * action, int offset)
     /*  */
 574 {
 575     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 576     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 577 
 578     if (action->type != action_type_rsc || target == NULL) {
 579         /* No limit on these */
 580         return;
 581     }
 582 
 583     /* if we have a router node, this means the action is performing
 584      * on a remote node. For now, we count all actions occurring on a
 585      * remote node against the job list on the cluster node hosting
 586      * the connection resources */
 587     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 588 
 589     if ((target == NULL) &&
 590         (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED))) {
 591 
 592         const char *t1 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
 593         const char *t2 = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 594 
 595         te_update_job_count_on(t1, offset, TRUE);
 596         te_update_job_count_on(t2, offset, TRUE);
 597         return;
 598     } else if (target == NULL) {
 599         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 600     }
 601 
 602     te_update_job_count_on(target, offset, FALSE);
 603 }
 604 
 605 static gboolean
 606 te_should_perform_action_on(crm_graph_t * graph, crm_action_t * action, const char *target)
     /*  */
 607 {
 608     int limit = 0;
 609     struct te_peer_s *r = NULL;
 610     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 611     const char *id = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 612 
 613     if(target == NULL) {
 614         /* No limit on these */
 615         return TRUE;
 616 
 617     } else if(te_targets == NULL) {
 618         return FALSE;
 619     }
 620 
 621     r = g_hash_table_lookup(te_targets, target);
 622     limit = throttle_get_job_limit(target);
 623 
 624     if(r == NULL) {
 625         r = calloc(1, sizeof(struct te_peer_s));
 626         r->name = strdup(target);
 627         g_hash_table_insert(te_targets, r->name, r);
 628     }
 629 
 630     if(limit <= r->jobs) {
 631         crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
 632                   target, limit, r->jobs, id);
 633         return FALSE;
 634 
 635     } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
 636         if (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED)) {
 637             crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
 638                       target, graph->migration_limit, r->migrate_jobs, id);
 639             return FALSE;
 640         }
 641     }
 642 
 643     crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);
 644 
 645     return TRUE;
 646 }
 647 
 648 static gboolean
 649 te_should_perform_action(crm_graph_t * graph, crm_action_t * action)
     /*  */
 650 {
 651     const char *target = NULL;
 652     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 653 
 654     if (action->type != action_type_rsc) {
 655         /* No limit on these */
 656         return TRUE;
 657     }
 658 
 659     /* if we have a router node, this means the action is performing
 660      * on a remote node. For now, we count all actions occurring on a
 661      * remote node against the job list on the cluster node hosting
 662      * the connection resources */
 663     target = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 664 
 665     if ((target == NULL) &&
 666         (safe_str_eq(task, CRMD_ACTION_MIGRATE) || safe_str_eq(task, CRMD_ACTION_MIGRATED))) {
 667 
 668         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_SOURCE);
 669         if(te_should_perform_action_on(graph, action, target) == FALSE) {
 670             return FALSE;
 671         }
 672 
 673         target = crm_meta_value(action->params, XML_LRM_ATTR_MIGRATE_TARGET);
 674 
 675     } else if (target == NULL) {
 676         target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 677     }
 678 
 679     return te_should_perform_action_on(graph, action, target);
 680 }
 681 
 682 void
 683 te_action_confirmed(crm_action_t * action)
     /*  */
 684 {
 685     const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 686 
 687     if (action->confirmed == FALSE && action->type == action_type_rsc && target != NULL) {
 688         te_update_job_count(action, -1);
 689     }
 690     action->confirmed = TRUE;
 691 }
 692 
 693 
 694 crm_graph_functions_t te_graph_fns = {
 695     te_pseudo_action,
 696     te_rsc_command,
 697     te_crm_command,
 698     te_fence_node,
 699     te_should_perform_action,
 700 };
 701 
 702 void
 703 notify_crmd(crm_graph_t * graph)
     /*  */
 704 {
 705     const char *type = "unknown";
 706     enum crmd_fsa_input event = I_NULL;
 707 
 708     crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));
 709 
 710     CRM_CHECK(graph->complete, graph->complete = TRUE);
 711 
 712     switch (graph->completion_action) {
 713         case tg_stop:
 714             type = "stop";
 715             if (fsa_state == S_TRANSITION_ENGINE) {
 716                 event = I_TE_SUCCESS;
 717             }
 718             break;
 719         case tg_done:
 720             type = "done";
 721             if (fsa_state == S_TRANSITION_ENGINE) {
 722                 event = I_TE_SUCCESS;
 723             }
 724             break;
 725 
 726         case tg_restart:
 727             type = "restart";
 728             if (fsa_state == S_TRANSITION_ENGINE) {
 729                 if (transition_timer->period_ms > 0) {
 730                     crm_timer_stop(transition_timer);
 731                     crm_timer_start(transition_timer);
 732                 } else {
 733                     event = I_PE_CALC;
 734                 }
 735 
 736             } else if (fsa_state == S_POLICY_ENGINE) {
 737                 register_fsa_action(A_PE_INVOKE);
 738             }
 739             break;
 740 
 741         case tg_shutdown:
 742             type = "shutdown";
 743             if (is_set(fsa_input_register, R_SHUTDOWN)) {
 744                 event = I_STOP;
 745 
 746             } else {
 747                 crm_err("We didn't ask to be shut down, yet our PE is telling us to.");
 748                 event = I_TERMINATE;
 749             }
 750     }
 751 
 752     crm_debug("Transition %d status: %s - %s", graph->id, type, crm_str(graph->abort_reason));
 753 
 754     graph->abort_reason = NULL;
 755     graph->completion_action = tg_done;
 756     clear_bit(fsa_input_register, R_IN_TRANSITION);
 757 
 758     if (event != I_NULL) {
 759         register_fsa_input(C_FSA_INTERNAL, event, NULL);
 760 
 761     } else if (fsa_source) {
 762         mainloop_set_trigger(fsa_source);
 763     }
 764 }

/* */