root/daemons/controld/controld_te_actions.c

/* */

DEFINITIONS

1. te_start_action_timer
2. execute_pseudo_action
3. get_target_rc
4. execute_cluster_action
5. synthesize_timeout_event
6. controld_record_action_event
7. controld_record_action_timeout
8. execute_rsc_action
9. te_peer_free
10. te_reset_job_counts
11. te_update_job_count_on
12. te_update_job_count
13. allowed_on_node
14. graph_action_allowed
15. te_action_confirmed
16. controld_register_graph_functions
17. notify_crmd

   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <crm/crm.h>
  14 #include <crm/cib.h>
  15 #include <crm/lrmd.h>               // lrmd_event_data_t, lrmd_free_event()
  16 #include <crm/common/xml.h>
  17 #include <crm/cluster.h>
  18 
  19 #include <pacemaker-internal.h>
  20 #include <pacemaker-controld.h>
  21 
  22 static GHashTable *te_targets = NULL;
  23 void send_rsc_command(pcmk__graph_action_t *action);
  24 static void te_update_job_count(pcmk__graph_action_t *action, int offset);
  25 
  26 static void
  27 te_start_action_timer(const pcmk__graph_t *graph, pcmk__graph_action_t *action)
     /*  */
  28 {
  29     action->timer = g_timeout_add(action->timeout + graph->network_delay,
  30                                   action_timer_callback, (void *) action);
  31     pcmk__assert(action->timer != 0);
  32 }
  33 
  34 /*!
  35  * \internal
  36  * \brief Execute a graph pseudo-action
  37  *
  38  * \param[in,out] graph   Transition graph being executed
  39  * \param[in,out] pseudo  Pseudo-action to execute
  40  *
  41  * \return Standard Pacemaker return code
  42  */
  43 static int
  44 execute_pseudo_action(pcmk__graph_t *graph, pcmk__graph_action_t *pseudo)
     /*  */
  45 {
  46     const char *task = crm_element_value(pseudo->xml, PCMK_XA_OPERATION);
  47 
  48     /* send to peers as well? */
  49     if (pcmk__str_eq(task, PCMK_ACTION_MAINTENANCE_NODES, pcmk__str_casei)) {
  50         GHashTableIter iter;
  51         crm_node_t *node = NULL;
  52 
  53         g_hash_table_iter_init(&iter, crm_peer_cache);
  54         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
  55             xmlNode *cmd = NULL;
  56 
  57             if (pcmk__str_eq(controld_globals.our_nodename, node->uname,
  58                              pcmk__str_casei)) {
  59                 continue;
  60             }
  61 
  62             cmd = create_request(task, pseudo->xml, node->uname,
  63                                  CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
  64             pcmk__cluster_send_message(node, crm_msg_crmd, cmd);
  65             free_xml(cmd);
  66         }
  67 
  68         remote_ra_process_maintenance_nodes(pseudo->xml);
  69     } else {
  70         /* Check action for Pacemaker Remote node side effects */
  71         remote_ra_process_pseudo(pseudo->xml);
  72     }
  73 
  74     crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
  75               crm_element_value(pseudo->xml, PCMK__XA_OPERATION_KEY));
  76     te_action_confirmed(pseudo, graph);
  77     return pcmk_rc_ok;
  78 }
  79 
  80 static int
  81 get_target_rc(pcmk__graph_action_t *action)
     /*  */
  82 {
  83     int exit_status;
  84 
  85     pcmk__scan_min_int(crm_meta_value(action->params, PCMK__META_OP_TARGET_RC),
  86                        &exit_status, 0);
  87     return exit_status;
  88 }
  89 
  90 /*!
  91  * \internal
  92  * \brief Execute a cluster action from a transition graph
  93  *
  94  * \param[in,out] graph   Transition graph being executed
  95  * \param[in,out] action  Cluster action to execute
  96  *
  97  * \return Standard Pacemaker return code
  98  */
  99 static int
 100 execute_cluster_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
     /*  */
 101 {
 102     char *counter = NULL;
 103     xmlNode *cmd = NULL;
 104     gboolean is_local = FALSE;
 105 
 106     const char *id = NULL;
 107     const char *task = NULL;
 108     const char *value = NULL;
 109     const char *on_node = NULL;
 110     const char *router_node = NULL;
 111 
 112     gboolean rc = TRUE;
 113     gboolean no_wait = FALSE;
 114 
 115     const crm_node_t *node = NULL;
 116 
 117     id = pcmk__xe_id(action->xml);
 118     CRM_CHECK(!pcmk__str_empty(id), return EPROTO);
 119 
 120     task = crm_element_value(action->xml, PCMK_XA_OPERATION);
 121     CRM_CHECK(!pcmk__str_empty(task), return EPROTO);
 122 
 123     on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
 124     CRM_CHECK(!pcmk__str_empty(on_node), return pcmk_rc_node_unknown);
 125 
 126     router_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
 127     if (router_node == NULL) {
 128         router_node = on_node;
 129         if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
 130             const char *mode = crm_element_value(action->xml, PCMK__XA_MODE);
 131 
 132             if (pcmk__str_eq(mode, PCMK__VALUE_CIB, pcmk__str_none)) {
 133                 router_node = controld_globals.our_nodename;
 134             }
 135         }
 136     }
 137 
 138     if (pcmk__str_eq(router_node, controld_globals.our_nodename,
 139                      pcmk__str_casei)) {
 140         is_local = TRUE;
 141     }
 142 
 143     value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT);
 144     if (crm_is_true(value)) {
 145         no_wait = TRUE;
 146     }
 147 
 148     crm_info("Handling controller request '%s' (%s on %s)%s%s",
 149              id, task, on_node, (is_local? " locally" : ""),
 150              (no_wait? " without waiting" : ""));
 151 
 152     if (is_local
 153         && pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
 154         /* defer until everything else completes */
 155         crm_info("Controller request '%s' is a local shutdown", id);
 156         graph->completion_action = pcmk__graph_shutdown;
 157         graph->abort_reason = "local shutdown";
 158         te_action_confirmed(action, graph);
 159         return pcmk_rc_ok;
 160 
 161     } else if (pcmk__str_eq(task, PCMK_ACTION_DO_SHUTDOWN, pcmk__str_none)) {
 162         crm_node_t *peer = pcmk__get_node(0, router_node, NULL,
 163                                           pcmk__node_search_cluster_member);
 164 
 165         pcmk__update_peer_expected(__func__, peer, CRMD_JOINSTATE_DOWN);
 166     }
 167 
 168     cmd = create_request(task, action->xml, router_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
 169 
 170     counter = pcmk__transition_key(controld_globals.transition_graph->id,
 171                                    action->id, get_target_rc(action),
 172                                    controld_globals.te_uuid);
 173     crm_xml_add(cmd, PCMK__XA_TRANSITION_KEY, counter);
 174 
 175     node = pcmk__get_node(0, router_node, NULL,
 176                           pcmk__node_search_cluster_member);
 177     rc = pcmk__cluster_send_message(node, crm_msg_crmd, cmd);
 178     free(counter);
 179     free_xml(cmd);
 180 
 181     if (rc == FALSE) {
 182         crm_err("Action %d failed: send", action->id);
 183         return ECOMM;
 184 
 185     } else if (no_wait) {
 186         te_action_confirmed(action, graph);
 187 
 188     } else {
 189         if (action->timeout <= 0) {
 190             crm_err("Action %d: %s on %s had an invalid timeout (%dms).  Using %ums instead",
 191                     action->id, task, on_node, action->timeout, graph->network_delay);
 192             action->timeout = (int) graph->network_delay;
 193         }
 194         te_start_action_timer(graph, action);
 195     }
 196 
 197     return pcmk_rc_ok;
 198 }
 199 
 200 /*!
 201  * \internal
 202  * \brief Synthesize an executor event for a resource action timeout
 203  *
 204  * \param[in] action     Resource action that timed out
 205  * \param[in] target_rc  Expected result of action that timed out
 206  *
 207  * Synthesize an executor event for a resource action timeout. (If the executor
 208  * gets a timeout while waiting for a resource action to complete, that will be
 209  * reported via the usual callback. This timeout means we didn't hear from the
 210  * executor itself or the controller that relayed the action to the executor.)
 211  *
 212  * \return Newly created executor event for result of \p action
 213  * \note The caller is responsible for freeing the return value using
 214  *       lrmd_free_event().
 215  */
 216 static lrmd_event_data_t *
 217 synthesize_timeout_event(const pcmk__graph_action_t *action, int target_rc)
     /*  */
 218 {
 219     lrmd_event_data_t *op = NULL;
 220     const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
 221     const char *reason = NULL;
 222     char *dynamic_reason = NULL;
 223 
 224     if (pcmk__str_eq(target, pcmk__cluster_local_node_name(),
 225                      pcmk__str_casei)) {
 226         reason = "Local executor did not return result in time";
 227     } else {
 228         const char *router_node = NULL;
 229 
 230         router_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
 231         if (router_node == NULL) {
 232             router_node = target;
 233         }
 234         dynamic_reason = crm_strdup_printf("Controller on %s did not return "
 235                                            "result in time", router_node);
 236         reason = dynamic_reason;
 237     }
 238 
 239     op = pcmk__event_from_graph_action(NULL, action, PCMK_EXEC_TIMEOUT,
 240                                        PCMK_OCF_UNKNOWN_ERROR, reason);
 241     op->call_id = -1;
 242     op->user_data = pcmk__transition_key(controld_globals.transition_graph->id,
 243                                          action->id, target_rc,
 244                                          controld_globals.te_uuid);
 245     free(dynamic_reason);
 246     return op;
 247 }
 248 
 249 static void
 250 controld_record_action_event(pcmk__graph_action_t *action,
     /*  */
 251                              lrmd_event_data_t *op)
 252 {
 253     cib_t *cib_conn = controld_globals.cib_conn;
 254 
 255     xmlNode *state = NULL;
 256     xmlNode *rsc = NULL;
 257     xmlNode *action_rsc = NULL;
 258 
 259     int rc = pcmk_ok;
 260 
 261     const char *rsc_id = NULL;
 262     const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
 263     const char *task_uuid = crm_element_value(action->xml,
 264                                               PCMK__XA_OPERATION_KEY);
 265     const char *target_uuid = crm_element_value(action->xml,
 266                                                 PCMK__META_ON_NODE_UUID);
 267 
 268     int target_rc = get_target_rc(action);
 269 
 270     action_rsc = pcmk__xe_first_child(action->xml, PCMK_XE_PRIMITIVE, NULL,
 271                                       NULL);
 272     if (action_rsc == NULL) {
 273         return;
 274     }
 275 
 276     rsc_id = pcmk__xe_id(action_rsc);
 277     CRM_CHECK(rsc_id != NULL,
 278               crm_log_xml_err(action->xml, "Bad:action"); return);
 279 
 280 /*
 281   update the CIB
 282 
 283 <node_state id="hadev">
 284       <lrm>
 285         <lrm_resources>
 286           <lrm_resource id="rsc2" last_op="start" op_code="0" target="hadev"/>
 287 */
 288 
 289     state = pcmk__xe_create(NULL, PCMK__XE_NODE_STATE);
 290 
 291     crm_xml_add(state, PCMK_XA_ID, target_uuid);
 292     crm_xml_add(state, PCMK_XA_UNAME, target);
 293 
 294     rsc = pcmk__xe_create(state, PCMK__XE_LRM);
 295     crm_xml_add(rsc, PCMK_XA_ID, target_uuid);
 296 
 297     rsc = pcmk__xe_create(rsc, PCMK__XE_LRM_RESOURCES);
 298     rsc = pcmk__xe_create(rsc, PCMK__XE_LRM_RESOURCE);
 299     crm_xml_add(rsc, PCMK_XA_ID, rsc_id);
 300 
 301 
 302     crm_copy_xml_element(action_rsc, rsc, PCMK_XA_TYPE);
 303     crm_copy_xml_element(action_rsc, rsc, PCMK_XA_CLASS);
 304     crm_copy_xml_element(action_rsc, rsc, PCMK_XA_PROVIDER);
 305 
 306     pcmk__create_history_xml(rsc, op, CRM_FEATURE_SET, target_rc, target,
 307                              __func__);
 308 
 309     rc = cib_conn->cmds->modify(cib_conn, PCMK_XE_STATUS, state,
 310                                 cib_scope_local);
 311     fsa_register_cib_callback(rc, NULL, cib_action_updated);
 312     free_xml(state);
 313 
 314     crm_trace("Sent CIB update (call ID %d) for synthesized event of action %d (%s on %s)",
 315               rc, action->id, task_uuid, target);
 316     pcmk__set_graph_action_flags(action, pcmk__graph_action_sent_update);
 317 }
 318 
 319 void
 320 controld_record_action_timeout(pcmk__graph_action_t *action)
     /*  */
 321 {
 322     lrmd_event_data_t *op = NULL;
 323 
 324     const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
 325     const char *task_uuid = crm_element_value(action->xml,
 326                                               PCMK__XA_OPERATION_KEY);
 327 
 328     int target_rc = get_target_rc(action);
 329 
 330     crm_warn("%s %d: %s on %s timed out",
 331              action->xml->name, action->id, task_uuid, target);
 332 
 333     op = synthesize_timeout_event(action, target_rc);
 334     controld_record_action_event(action, op);
 335     lrmd_free_event(op);
 336 }
 337 
 338 /*!
 339  * \internal
 340  * \brief Execute a resource action from a transition graph
 341  *
 342  * \param[in,out] graph   Transition graph being executed
 343  * \param[in,out] action  Resource action to execute
 344  *
 345  * \return Standard Pacemaker return code
 346  */
 347 static int
 348 execute_rsc_action(pcmk__graph_t *graph, pcmk__graph_action_t *action)
     /*  */
 349 {
 350     /* never overwrite stop actions in the CIB with
 351      *   anything other than completed results
 352      *
 353      * Writing pending stops makes it look like the
 354      *   resource is running again
 355      */
 356     xmlNode *cmd = NULL;
 357     xmlNode *rsc_op = NULL;
 358 
 359     gboolean rc = TRUE;
 360     gboolean no_wait = FALSE;
 361     gboolean is_local = FALSE;
 362 
 363     char *counter = NULL;
 364     const char *task = NULL;
 365     const char *value = NULL;
 366     const char *on_node = NULL;
 367     const char *router_node = NULL;
 368     const char *task_uuid = NULL;
 369 
 370     pcmk__assert((action != NULL) && (action->xml != NULL));
 371 
 372     pcmk__clear_graph_action_flags(action, pcmk__graph_action_executed);
 373     on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
 374 
 375     CRM_CHECK(!pcmk__str_empty(on_node),
 376               crm_err("Corrupted command(id=%s) %s: no node",
 377                       pcmk__xe_id(action->xml), pcmk__s(task, "without task"));
 378               return pcmk_rc_node_unknown);
 379 
 380     rsc_op = action->xml;
 381     task = crm_element_value(rsc_op, PCMK_XA_OPERATION);
 382     task_uuid = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY);
 383     router_node = crm_element_value(rsc_op, PCMK__XA_ROUTER_NODE);
 384 
 385     if (!router_node) {
 386         router_node = on_node;
 387     }
 388 
 389     counter = pcmk__transition_key(controld_globals.transition_graph->id,
 390                                    action->id, get_target_rc(action),
 391                                    controld_globals.te_uuid);
 392     crm_xml_add(rsc_op, PCMK__XA_TRANSITION_KEY, counter);
 393 
 394     if (pcmk__str_eq(router_node, controld_globals.our_nodename,
 395                      pcmk__str_casei)) {
 396         is_local = TRUE;
 397     }
 398 
 399     value = crm_meta_value(action->params, PCMK__META_OP_NO_WAIT);
 400     if (crm_is_true(value)) {
 401         no_wait = TRUE;
 402     }
 403 
 404     crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
 405                task, task_uuid, (is_local? " locally" : ""), on_node,
 406                (no_wait? " without waiting" : ""), action->id);
 407 
 408     cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
 409                          CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
 410 
 411     if (is_local) {
 412         /* shortcut local resource commands */
 413         ha_msg_input_t data = {
 414             .msg = cmd,
 415             .xml = rsc_op,
 416         };
 417 
 418         fsa_data_t msg = {
 419             .id = 0,
 420             .data = &data,
 421             .data_type = fsa_dt_ha_msg,
 422             .fsa_input = I_NULL,
 423             .fsa_cause = C_FSA_INTERNAL,
 424             .actions = A_LRM_INVOKE,
 425             .origin = __func__,
 426         };
 427 
 428         do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, controld_globals.fsa_state,
 429                       I_NULL, &msg);
 430 
 431     } else {
 432         const crm_node_t *node =
 433             pcmk__get_node(0, router_node, NULL,
 434                            pcmk__node_search_cluster_member);
 435 
 436         rc = pcmk__cluster_send_message(node, crm_msg_lrmd, cmd);
 437     }
 438 
 439     free(counter);
 440     free_xml(cmd);
 441 
 442     pcmk__set_graph_action_flags(action, pcmk__graph_action_executed);
 443 
 444     if (rc == FALSE) {
 445         crm_err("Action %d failed: send", action->id);
 446         return ECOMM;
 447 
 448     } else if (no_wait) {
 449         /* Just mark confirmed. Don't bump the job count only to immediately
 450          * decrement it.
 451          */
 452         crm_info("Action %d confirmed - no wait", action->id);
 453         pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
 454         pcmk__update_graph(controld_globals.transition_graph, action);
 455         trigger_graph();
 456 
 457     } else if (pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
 458         crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
 459                   action->id, task, task_uuid, on_node, action->timeout);
 460     } else {
 461         if (action->timeout <= 0) {
 462             crm_err("Action %d: %s %s on %s had an invalid timeout (%dms).  Using %ums instead",
 463                     action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
 464             action->timeout = (int) graph->network_delay;
 465         }
 466         te_update_job_count(action, 1);
 467         te_start_action_timer(graph, action);
 468     }
 469 
 470     return pcmk_rc_ok;
 471 }
 472 
 473 struct te_peer_s
 474 {
 475         char *name;
 476         int jobs;
 477         int migrate_jobs;
 478 };
 479 
 480 static void te_peer_free(gpointer p)
     /*  */
 481 {
 482     struct te_peer_s *peer = p;
 483 
 484     free(peer->name);
 485     free(peer);
 486 }
 487 
 488 void te_reset_job_counts(void)
     /*  */
 489 {
 490     GHashTableIter iter;
 491     struct te_peer_s *peer = NULL;
 492 
 493     if(te_targets == NULL) {
 494         te_targets = pcmk__strkey_table(NULL, te_peer_free);
 495     }
 496 
 497     g_hash_table_iter_init(&iter, te_targets);
 498     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & peer)) {
 499         peer->jobs = 0;
 500         peer->migrate_jobs = 0;
 501     }
 502 }
 503 
 504 static void
 505 te_update_job_count_on(const char *target, int offset, bool migrate)
     /*  */
 506 {
 507     struct te_peer_s *r = NULL;
 508 
 509     if(target == NULL || te_targets == NULL) {
 510         return;
 511     }
 512 
 513     r = g_hash_table_lookup(te_targets, target);
 514     if(r == NULL) {
 515         r = pcmk__assert_alloc(1, sizeof(struct te_peer_s));
 516         r->name = pcmk__str_copy(target);
 517         g_hash_table_insert(te_targets, r->name, r);
 518     }
 519 
 520     r->jobs += offset;
 521     if(migrate) {
 522         r->migrate_jobs += offset;
 523     }
 524     crm_trace("jobs[%s] = %d", target, r->jobs);
 525 }
 526 
 527 static void
 528 te_update_job_count(pcmk__graph_action_t *action, int offset)
     /*  */
 529 {
 530     const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
 531     const char *target = crm_element_value(action->xml, PCMK__META_ON_NODE);
 532 
 533     if ((action->type != pcmk__rsc_graph_action) || (target == NULL)) {
 534         /* No limit on these */
 535         return;
 536     }
 537 
 538     /* if we have a router node, this means the action is performing
 539      * on a remote node. For now, we count all actions occurring on a
 540      * remote node against the job list on the cluster node hosting
 541      * the connection resources */
 542     target = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
 543 
 544     if ((target == NULL)
 545         && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
 546                                 PCMK_ACTION_MIGRATE_FROM, NULL)) {
 547 
 548         const char *t1 = crm_meta_value(action->params,
 549                                         PCMK__META_MIGRATE_SOURCE);
 550         const char *t2 = crm_meta_value(action->params,
 551                                         PCMK__META_MIGRATE_TARGET);
 552 
 553         te_update_job_count_on(t1, offset, TRUE);
 554         te_update_job_count_on(t2, offset, TRUE);
 555         return;
 556     } else if (target == NULL) {
 557         target = crm_element_value(action->xml, PCMK__META_ON_NODE);
 558     }
 559 
 560     te_update_job_count_on(target, offset, FALSE);
 561 }
 562 
 563 /*!
 564  * \internal
 565  * \brief Check whether a graph action is allowed to be executed on a node
 566  *
 567  * \param[in] graph   Transition graph being executed
 568  * \param[in] action  Graph action being executed
 569  * \param[in] target  Name of node where action should be executed
 570  *
 571  * \return true if action is allowed, otherwise false
 572  */
 573 static bool
 574 allowed_on_node(const pcmk__graph_t *graph, const pcmk__graph_action_t *action,
     /*  */
 575                 const char *target)
 576 {
 577     int limit = 0;
 578     struct te_peer_s *r = NULL;
 579     const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
 580     const char *id = crm_element_value(action->xml, PCMK__XA_OPERATION_KEY);
 581 
 582     if(target == NULL) {
 583         /* No limit on these */
 584         return true;
 585 
 586     } else if(te_targets == NULL) {
 587         return false;
 588     }
 589 
 590     r = g_hash_table_lookup(te_targets, target);
 591     limit = throttle_get_job_limit(target);
 592 
 593     if(r == NULL) {
 594         r = pcmk__assert_alloc(1, sizeof(struct te_peer_s));
 595         r->name = pcmk__str_copy(target);
 596         g_hash_table_insert(te_targets, r->name, r);
 597     }
 598 
 599     if(limit <= r->jobs) {
 600         crm_trace("Peer %s is over their job limit of %d (%d): deferring %s",
 601                   target, limit, r->jobs, id);
 602         return false;
 603 
 604     } else if(graph->migration_limit > 0 && r->migrate_jobs >= graph->migration_limit) {
 605         if (pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
 606                                  PCMK_ACTION_MIGRATE_FROM, NULL)) {
 607             crm_trace("Peer %s is over their migration job limit of %d (%d): deferring %s",
 608                       target, graph->migration_limit, r->migrate_jobs, id);
 609             return false;
 610         }
 611     }
 612 
 613     crm_trace("Peer %s has not hit their limit yet. current jobs = %d limit= %d limit", target, r->jobs, limit);
 614 
 615     return true;
 616 }
 617 
 618 /*!
 619  * \internal
 620  * \brief Check whether a graph action is allowed to be executed
 621  *
 622  * \param[in] graph   Transition graph being executed
 623  * \param[in] action  Graph action being executed
 624  *
 625  * \return true if action is allowed, otherwise false
 626  */
 627 static bool
 628 graph_action_allowed(pcmk__graph_t *graph, pcmk__graph_action_t *action)
     /*  */
 629 {
 630     const char *target = NULL;
 631     const char *task = crm_element_value(action->xml, PCMK_XA_OPERATION);
 632 
 633     if (action->type != pcmk__rsc_graph_action) {
 634         /* No limit on these */
 635         return true;
 636     }
 637 
 638     /* if we have a router node, this means the action is performing
 639      * on a remote node. For now, we count all actions occurring on a
 640      * remote node against the job list on the cluster node hosting
 641      * the connection resources */
 642     target = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
 643 
 644     if ((target == NULL)
 645         && pcmk__strcase_any_of(task, PCMK_ACTION_MIGRATE_TO,
 646                                 PCMK_ACTION_MIGRATE_FROM, NULL)) {
 647         target = crm_meta_value(action->params, PCMK__META_MIGRATE_SOURCE);
 648         if (!allowed_on_node(graph, action, target)) {
 649             return false;
 650         }
 651 
 652         target = crm_meta_value(action->params, PCMK__META_MIGRATE_TARGET);
 653 
 654     } else if (target == NULL) {
 655         target = crm_element_value(action->xml, PCMK__META_ON_NODE);
 656     }
 657 
 658     return allowed_on_node(graph, action, target);
 659 }
 660 
 661 /*!
 662  * \brief Confirm a graph action (and optionally update graph)
 663  *
 664  * \param[in,out] action  Action to confirm
 665  * \param[in,out] graph   Update and trigger this graph (if non-NULL)
 666  */
 667 void
 668 te_action_confirmed(pcmk__graph_action_t *action, pcmk__graph_t *graph)
     /*  */
 669 {
 670     if (!pcmk_is_set(action->flags, pcmk__graph_action_confirmed)) {
 671         if ((action->type == pcmk__rsc_graph_action)
 672             && (crm_element_value(action->xml, PCMK__META_ON_NODE) != NULL)) {
 673             te_update_job_count(action, -1);
 674         }
 675         pcmk__set_graph_action_flags(action, pcmk__graph_action_confirmed);
 676     }
 677     if (graph) {
 678         pcmk__update_graph(graph, action);
 679         trigger_graph();
 680     }
 681 }
 682 
 683 static pcmk__graph_functions_t te_graph_fns = {
 684     execute_pseudo_action,
 685     execute_rsc_action,
 686     execute_cluster_action,
 687     controld_execute_fence_action,
 688     graph_action_allowed,
 689 };
 690 
 691 /*
 692  * \internal
 693  * \brief Register the transitioner's graph functions with \p libpacemaker
 694  */
 695 void
 696 controld_register_graph_functions(void)
     /*  */
 697 {
 698     pcmk__set_graph_functions(&te_graph_fns);
 699 }
 700 
 701 void
 702 notify_crmd(pcmk__graph_t *graph)
     /*  */
 703 {
 704     const char *type = "unknown";
 705     enum crmd_fsa_input event = I_NULL;
 706 
 707     crm_debug("Processing transition completion in state %s",
 708               fsa_state2string(controld_globals.fsa_state));
 709 
 710     CRM_CHECK(graph->complete, graph->complete = true);
 711 
 712     switch (graph->completion_action) {
 713         case pcmk__graph_wait:
 714             type = "stop";
 715             if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
 716                 event = I_TE_SUCCESS;
 717             }
 718             break;
 719         case pcmk__graph_done:
 720             type = "done";
 721             if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
 722                 event = I_TE_SUCCESS;
 723             }
 724             break;
 725 
 726         case pcmk__graph_restart:
 727             type = "restart";
 728             if (controld_globals.fsa_state == S_TRANSITION_ENGINE) {
 729                 if (controld_get_period_transition_timer() > 0) {
 730                     controld_stop_transition_timer();
 731                     controld_start_transition_timer();
 732                 } else {
 733                     event = I_PE_CALC;
 734                 }
 735 
 736             } else if (controld_globals.fsa_state == S_POLICY_ENGINE) {
 737                 controld_set_fsa_action_flags(A_PE_INVOKE);
 738                 controld_trigger_fsa();
 739             }
 740             break;
 741 
 742         case pcmk__graph_shutdown:
 743             type = "shutdown";
 744             if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 745                 event = I_STOP;
 746 
 747             } else {
 748                 crm_err("We didn't ask to be shut down, yet the scheduler is telling us to");
 749                 event = I_TERMINATE;
 750             }
 751     }
 752 
 753     crm_debug("Transition %d status: %s - %s", graph->id, type,
 754               pcmk__s(graph->abort_reason, "unspecified reason"));
 755 
 756     graph->abort_reason = NULL;
 757     graph->completion_action = pcmk__graph_done;
 758 
 759     if (event != I_NULL) {
 760         register_fsa_input(C_FSA_INTERNAL, event, NULL);
 761     } else {
 762         controld_trigger_fsa();
 763     }
 764 }

/* */