daemons/controld/controld

/* */
This source file includes following definitions.
load2str
find_cib_loadfile
throttle_cib_load
throttle_load_avg
throttle_check_thresholds
throttle_handle_load
throttle_mode
throttle_send_command
throttle_timer_cb
throttle_record_free
throttle_set_load_target
throttle_update_job_max
throttle_init
throttle_fini
throttle_get_total_job_limit
throttle_get_job_limit
throttle_update
   1 /*
   2  * Copyright 2013-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/types.h>
  13 #include <sys/stat.h>
  14 
  15 #include <unistd.h>
  16 #include <ctype.h>
  17 #include <dirent.h>
  18 
  19 #include <crm/crm.h>
  20 #include <crm/msg_xml.h>
  21 #include <crm/cluster.h>
  22 
  23 #include <pacemaker-controld.h>
  24 
  25 /* These values don't need to be bits, but these particular values must be kept
  26  * for backward compatibility during rolling upgrades.
  27  */
  28 enum throttle_state_e {
  29     throttle_none       = 0x0000,
  30     throttle_low        = 0x0001,
  31     throttle_med        = 0x0010,
  32     throttle_high       = 0x0100,
  33     throttle_extreme    = 0x1000,
  34 };
  35 
  36 struct throttle_record_s {
  37     int max;
  38     enum throttle_state_e mode;
  39     char *node;
  40 };
  41 
  42 static int throttle_job_max = 0;
  43 static float throttle_load_target = 0.0;
  44 
  45 #define THROTTLE_FACTOR_LOW    1.2
  46 #define THROTTLE_FACTOR_MEDIUM 1.6
  47 #define THROTTLE_FACTOR_HIGH   2.0
  48 
  49 static GHashTable *throttle_records = NULL;
  50 static mainloop_timer_t *throttle_timer = NULL;
  51 
  52 static const char *
  53 load2str(enum throttle_state_e mode)
     /*  */
  54 {
  55     switch (mode) {
  56         case throttle_extreme:  return "extreme";
  57         case throttle_high:     return "high";
  58         case throttle_med:      return "medium";
  59         case throttle_low:      return "low";
  60         case throttle_none:     return "negligible";
  61         default:                return "undetermined";
  62     }
  63 }
  64 
  65 #if SUPPORT_PROCFS
  66 /*!
  67  * \internal
  68  * \brief Return name of /proc file containing the CIB daemon's load statistics
  69  *
  70  * \return Newly allocated memory with file name on success, NULL otherwise
  71  *
  72  * \note It is the caller's responsibility to free the return value.
  73  *       This will return NULL if the daemon is being run via valgrind.
  74  *       This should be called only on Linux systems.
  75  */
  76 static char *
  77 find_cib_loadfile(void)
     /*  */
  78 {
  79     pid_t pid = pcmk__procfs_pid_of("pacemaker-based");
  80 
  81     return pid? crm_strdup_printf("/proc/%lld/stat", (long long) pid) : NULL;
  82 }
  83 
  84 static bool
  85 throttle_cib_load(float *load)
     /*  */
  86 {
  87 /*
  88        /proc/[pid]/stat
  89               Status information about the process.  This is used by ps(1).  It is defined in /usr/src/linux/fs/proc/array.c.
  90 
  91               The fields, in order, with their proper scanf(3) format specifiers, are:
  92 
  93               pid %d      (1) The process ID.
  94 
  95               comm %s     (2) The filename of the executable, in parentheses.  This is visible whether or not the executable is swapped out.
  96 
  97               state %c    (3) One character from the string "RSDZTW" where R is running, S is sleeping in an interruptible wait, D is waiting in uninterruptible disk sleep, Z is zombie, T is traced or stopped (on a signal), and W is paging.
  98 
  99               ppid %d     (4) The PID of the parent.
 100 
 101               pgrp %d     (5) The process group ID of the process.
 102 
 103               session %d  (6) The session ID of the process.
 104 
 105               tty_nr %d   (7) The controlling terminal of the process.  (The minor device number is contained in the combination of bits 31 to 20 and 7 to 0; the major device number is in bits 15 to 8.)
 106 
 107               tpgid %d    (8) The ID of the foreground process group of the controlling terminal of the process.
 108 
 109               flags %u (%lu before Linux 2.6.22)
 110                           (9) The kernel flags word of the process.  For bit meanings, see the PF_* defines in the Linux kernel source file include/linux/sched.h.  Details depend on the kernel version.
 111 
 112               minflt %lu  (10) The number of minor faults the process has made which have not required loading a memory page from disk.
 113 
 114               cminflt %lu (11) The number of minor faults that the process's waited-for children have made.
 115 
 116               majflt %lu  (12) The number of major faults the process has made which have required loading a memory page from disk.
 117 
 118               cmajflt %lu (13) The number of major faults that the process's waited-for children have made.
 119 
 120               utime %lu   (14) Amount of time that this process has been scheduled in user mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)).  This includes guest time, guest_time (time spent running a virtual CPU, see below), so that applications that are not aware of the guest time field do not lose that time from their calculations.
 121 
 122               stime %lu   (15) Amount of time that this process has been scheduled in kernel mode, measured in clock ticks (divide by sysconf(_SC_CLK_TCK)).
 123  */
 124 
 125     static char *loadfile = NULL;
 126     static time_t last_call = 0;
 127     static long ticks_per_s = 0;
 128     static unsigned long last_utime, last_stime;
 129 
 130     char buffer[64*1024];
 131     FILE *stream = NULL;
 132     time_t now = time(NULL);
 133 
 134     if(load == NULL) {
 135         return FALSE;
 136     } else {
 137         *load = 0.0;
 138     }
 139 
 140     if(loadfile == NULL) {
 141         last_call = 0;
 142         last_utime = 0;
 143         last_stime = 0;
 144         loadfile = find_cib_loadfile();
 145         if (loadfile == NULL) {
 146             crm_warn("Couldn't find CIB load file");
 147             return FALSE;
 148         }
 149         ticks_per_s = sysconf(_SC_CLK_TCK);
 150         crm_trace("Found %s", loadfile);
 151     }
 152 
 153     stream = fopen(loadfile, "r");
 154     if(stream == NULL) {
 155         int rc = errno;
 156 
 157         crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_strerror(rc), rc);
 158         free(loadfile); loadfile = NULL;
 159         return FALSE;
 160     }
 161 
 162     if(fgets(buffer, sizeof(buffer), stream)) {
 163         char *comm = calloc(1, 256);
 164         char state = 0;
 165         int rc = 0, pid = 0, ppid = 0, pgrp = 0, session = 0, tty_nr = 0, tpgid = 0;
 166         unsigned long flags = 0, minflt = 0, cminflt = 0, majflt = 0, cmajflt = 0, utime = 0, stime = 0;
 167 
 168         rc = sscanf(buffer,  "%d %[^ ] %c %d %d %d %d %d %lu %lu %lu %lu %lu %lu %lu",
 169                     &pid, comm, &state,
 170                     &ppid, &pgrp, &session, &tty_nr, &tpgid,
 171                     &flags, &minflt, &cminflt, &majflt, &cmajflt, &utime, &stime);
 172         free(comm);
 173 
 174         if(rc != 15) {
 175             crm_err("Only %d of 15 fields found in %s", rc, loadfile);
 176             fclose(stream);
 177             return FALSE;
 178 
 179         } else if(last_call > 0
 180            && last_call < now
 181            && last_utime <= utime
 182            && last_stime <= stime) {
 183 
 184             time_t elapsed = now - last_call;
 185             unsigned long delta_utime = utime - last_utime;
 186             unsigned long delta_stime = stime - last_stime;
 187 
 188             *load = (delta_utime + delta_stime); /* Cast to a float before division */
 189             *load /= ticks_per_s;
 190             *load /= elapsed;
 191             crm_debug("cib load: %f (%lu ticks in %lds)", *load, delta_utime + delta_stime, (long)elapsed);
 192 
 193         } else {
 194             crm_debug("Init %lu + %lu ticks at %ld (%lu tps)", utime, stime, (long)now, ticks_per_s);
 195         }
 196 
 197         last_call = now;
 198         last_utime = utime;
 199         last_stime = stime;
 200 
 201         fclose(stream);
 202         return TRUE;
 203     }
 204 
 205     fclose(stream);
 206     return FALSE;
 207 }
 208 
 209 static bool
 210 throttle_load_avg(float *load)
     /*  */
 211 {
 212     char buffer[256];
 213     FILE *stream = NULL;
 214     const char *loadfile = "/proc/loadavg";
 215 
 216     if(load == NULL) {
 217         return FALSE;
 218     }
 219 
 220     stream = fopen(loadfile, "r");
 221     if(stream == NULL) {
 222         int rc = errno;
 223         crm_warn("Couldn't read %s: %s (%d)", loadfile, pcmk_strerror(rc), rc);
 224         return FALSE;
 225     }
 226 
 227     if(fgets(buffer, sizeof(buffer), stream)) {
 228         char *nl = strstr(buffer, "\n");
 229 
 230         /* Grab the 1-minute average, ignore the rest */
 231         *load = strtof(buffer, NULL);
 232         if(nl) { nl[0] = 0; }
 233 
 234         fclose(stream);
 235         return TRUE;
 236     }
 237 
 238     fclose(stream);
 239     return FALSE;
 240 }
 241 
 242 /*!
 243  * \internal
 244  * \brief Check a load value against throttling thresholds
 245  *
 246  * \param[in] load        Load value to check
 247  * \param[in] desc        Description of metric (for logging)
 248  * \param[in] thresholds  Low/medium/high/extreme thresholds
 249  *
 250  * \return Throttle mode corresponding to load value
 251  */
 252 static enum throttle_state_e
 253 throttle_check_thresholds(float load, const char *desc, float thresholds[4])
     /*  */
 254 {
 255     if (load > thresholds[3]) {
 256         crm_notice("Extreme %s detected: %f", desc, load);
 257         return throttle_extreme;
 258 
 259     } else if (load > thresholds[2]) {
 260         crm_notice("High %s detected: %f", desc, load);
 261         return throttle_high;
 262 
 263     } else if (load > thresholds[1]) {
 264         crm_info("Moderate %s detected: %f", desc, load);
 265         return throttle_med;
 266 
 267     } else if (load > thresholds[0]) {
 268         crm_debug("Noticeable %s detected: %f", desc, load);
 269         return throttle_low;
 270     }
 271 
 272     crm_trace("Negligible %s detected: %f", desc, load);
 273     return throttle_none;
 274 }
 275 
 276 static enum throttle_state_e
 277 throttle_handle_load(float load, const char *desc, int cores)
     /*  */
 278 {
 279     float normalize;
 280     float thresholds[4];
 281 
 282     if (cores == 1) {
 283         /* On a single core machine, a load of 1.0 is already too high */
 284         normalize = 0.6;
 285 
 286     } else {
 287         /* Normalize the load to be per-core */
 288         normalize = cores;
 289     }
 290     thresholds[0] = throttle_load_target * normalize * THROTTLE_FACTOR_LOW;
 291     thresholds[1] = throttle_load_target * normalize * THROTTLE_FACTOR_MEDIUM;
 292     thresholds[2] = throttle_load_target * normalize * THROTTLE_FACTOR_HIGH;
 293     thresholds[3] = load + 1.0; /* never extreme */
 294 
 295     return throttle_check_thresholds(load, desc, thresholds);
 296 }
 297 #endif
 298 
 299 static enum throttle_state_e
 300 throttle_mode(void)
     /*  */
 301 {
 302     enum throttle_state_e mode = throttle_none;
 303 
 304 #if SUPPORT_PROCFS
 305     unsigned int cores;
 306     float load;
 307     float thresholds[4];
 308 
 309     cores = pcmk__procfs_num_cores();
 310     if(throttle_cib_load(&load)) {
 311         float cib_max_cpu = 0.95;
 312 
 313         /* The CIB is a single-threaded task and thus cannot consume
 314          * more than 100% of a CPU (and 1/cores of the overall system
 315          * load).
 316          *
 317          * On a many-cored system, the CIB might therefore be maxed out
 318          * (causing operations to fail or appear to fail) even though
 319          * the overall system load is still reasonable.
 320          *
 321          * Therefore, the 'normal' thresholds can not apply here, and we
 322          * need a special case.
 323          */
 324         if(cores == 1) {
 325             cib_max_cpu = 0.4;
 326         }
 327         if(throttle_load_target > 0.0 && throttle_load_target < cib_max_cpu) {
 328             cib_max_cpu = throttle_load_target;
 329         }
 330 
 331         thresholds[0] = cib_max_cpu * 0.8;
 332         thresholds[1] = cib_max_cpu * 0.9;
 333         thresholds[2] = cib_max_cpu;
 334         /* Can only happen on machines with a low number of cores */
 335         thresholds[3] = cib_max_cpu * 1.5;
 336 
 337         mode = throttle_check_thresholds(load, "CIB load", thresholds);
 338     }
 339 
 340     if(throttle_load_target <= 0) {
 341         /* If we ever make this a valid value, the cluster will at least behave as expected */
 342         return mode;
 343     }
 344 
 345     if(throttle_load_avg(&load)) {
 346         enum throttle_state_e cpu_load;
 347 
 348         cpu_load = throttle_handle_load(load, "CPU load", cores);
 349         if (cpu_load > mode) {
 350             mode = cpu_load;
 351         }
 352         crm_debug("Current load is %f across %u core(s)", load, cores);
 353     }
 354 #endif // SUPPORT_PROCFS
 355     return mode;
 356 }
 357 
 358 static void
 359 throttle_send_command(enum throttle_state_e mode)
     /*  */
 360 {
 361     xmlNode *xml = NULL;
 362     static enum throttle_state_e last = -1;
 363 
 364     if(mode != last) {
 365         crm_info("New throttle mode: %s load (was %s)",
 366                  load2str(mode), load2str(last));
 367         last = mode;
 368 
 369         xml = create_request(CRM_OP_THROTTLE, NULL, NULL, CRM_SYSTEM_CRMD, CRM_SYSTEM_CRMD, NULL);
 370         crm_xml_add_int(xml, F_CRM_THROTTLE_MODE, mode);
 371         crm_xml_add_int(xml, F_CRM_THROTTLE_MAX, throttle_job_max);
 372 
 373         send_cluster_message(NULL, crm_msg_crmd, xml, TRUE);
 374         free_xml(xml);
 375     }
 376 }
 377 
 378 static gboolean
 379 throttle_timer_cb(gpointer data)
     /*  */
 380 {
 381     throttle_send_command(throttle_mode());
 382     return TRUE;
 383 }
 384 
 385 static void
 386 throttle_record_free(gpointer p)
     /*  */
 387 {
 388     struct throttle_record_s *r = p;
 389     free(r->node);
 390     free(r);
 391 }
 392 
 393 void
 394 throttle_set_load_target(float target)
     /*  */
 395 {
 396     throttle_load_target = target;
 397 }
 398 
 399 /*!
 400  * \internal
 401  * \brief Update the maximum number of simultaneous jobs
 402  *
 403  * \param[in] preference  Cluster-wide node-action-limit from the CIB
 404  */
 405 void
 406 throttle_update_job_max(const char *preference)
     /*  */
 407 {
 408     long long max = 0LL;
 409     const char *env_limit = getenv("PCMK_node_action_limit");
 410 
 411     if (env_limit != NULL) {
 412         preference = env_limit; // Per-node override
 413     }
 414     if (preference != NULL) {
 415         pcmk__scan_ll(preference, &max, 0LL);
 416     }
 417     if (max > 0) {
 418         throttle_job_max = (int) max;
 419     } else {
 420         // Default is based on the number of cores detected
 421         throttle_job_max = 2 * pcmk__procfs_num_cores();
 422     }
 423 }
 424 
 425 void
 426 throttle_init(void)
     /*  */
 427 {
 428     if(throttle_records == NULL) {
 429         throttle_records = pcmk__strkey_table(NULL, throttle_record_free);
 430         throttle_timer = mainloop_timer_add("throttle", 30 * 1000, TRUE, throttle_timer_cb, NULL);
 431     }
 432 
 433     throttle_update_job_max(NULL);
 434     mainloop_timer_start(throttle_timer);
 435 }
 436 
 437 void
 438 throttle_fini(void)
     /*  */
 439 {
 440     if (throttle_timer != NULL) {
 441         mainloop_timer_del(throttle_timer);
 442         throttle_timer = NULL;
 443     }
 444     if (throttle_records != NULL) {
 445         g_hash_table_destroy(throttle_records);
 446         throttle_records = NULL;
 447     }
 448 }
 449 
 450 int
 451 throttle_get_total_job_limit(int l)
     /*  */
 452 {
 453     /* Cluster-wide limit */
 454     GHashTableIter iter;
 455     int limit = l;
 456     int peers = crm_active_peers();
 457     struct throttle_record_s *r = NULL;
 458 
 459     g_hash_table_iter_init(&iter, throttle_records);
 460 
 461     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &r)) {
 462         switch(r->mode) {
 463 
 464             case throttle_extreme:
 465                 if(limit == 0 || limit > peers/4) {
 466                     limit = QB_MAX(1, peers/4);
 467                 }
 468                 break;
 469 
 470             case throttle_high:
 471                 if(limit == 0 || limit > peers/2) {
 472                     limit = QB_MAX(1, peers/2);
 473                 }
 474                 break;
 475             default:
 476                 break;
 477         }
 478     }
 479     if(limit == l) {
 480         /* crm_trace("No change to batch-limit=%d", limit); */
 481 
 482     } else if(l == 0) {
 483         crm_trace("Using batch-limit=%d", limit);
 484 
 485     } else {
 486         crm_trace("Using batch-limit=%d instead of %d", limit, l);
 487     }
 488     return limit;
 489 }
 490 
 491 int
 492 throttle_get_job_limit(const char *node)
     /*  */
 493 {
 494     int jobs = 1;
 495     struct throttle_record_s *r = NULL;
 496 
 497     r = g_hash_table_lookup(throttle_records, node);
 498     if(r == NULL) {
 499         r = calloc(1, sizeof(struct throttle_record_s));
 500         r->node = strdup(node);
 501         r->mode = throttle_low;
 502         r->max = throttle_job_max;
 503         crm_trace("Defaulting to local values for unknown node %s", node);
 504 
 505         g_hash_table_insert(throttle_records, r->node, r);
 506     }
 507 
 508     switch(r->mode) {
 509         case throttle_extreme:
 510         case throttle_high:
 511             jobs = 1; /* At least one job must always be allowed */
 512             break;
 513         case throttle_med:
 514             jobs = QB_MAX(1, r->max / 4);
 515             break;
 516         case throttle_low:
 517             jobs = QB_MAX(1, r->max / 2);
 518             break;
 519         case throttle_none:
 520             jobs = QB_MAX(1, r->max);
 521             break;
 522         default:
 523             crm_err("Unknown throttle mode %.4x on %s", r->mode, node);
 524             break;
 525     }
 526     return jobs;
 527 }
 528 
 529 void
 530 throttle_update(xmlNode *xml)
     /*  */
 531 {
 532     int max = 0;
 533     int mode = 0;
 534     struct throttle_record_s *r = NULL;
 535     const char *from = crm_element_value(xml, F_CRM_HOST_FROM);
 536 
 537     crm_element_value_int(xml, F_CRM_THROTTLE_MODE, &mode);
 538     crm_element_value_int(xml, F_CRM_THROTTLE_MAX, &max);
 539 
 540     r = g_hash_table_lookup(throttle_records, from);
 541 
 542     if(r == NULL) {
 543         r = calloc(1, sizeof(struct throttle_record_s));
 544         r->node = strdup(from);
 545         g_hash_table_insert(throttle_records, r->node, r);
 546     }
 547 
 548     r->max = max;
 549     r->mode = (enum throttle_state_e) mode;
 550 
 551     crm_debug("Node %s has %s load and supports at most %d jobs; new job limit %d",
 552               from, load2str((enum throttle_state_e) mode), max,
 553               throttle_get_job_limit(from));
 554 }
/* */
root/daemons/controld/controld_throttle.c

DEFINITIONS
