/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * ident "%Z%%M% %I% %E% SMI" */ package com.sun.solaris.domain.pools; import java.io.*; import java.util.*; import java.util.logging.*; import java.text.DecimalFormat; import java.util.concurrent.atomic.*; import com.sun.solaris.service.locality.*; import com.sun.solaris.service.logging.*; import com.sun.solaris.service.pools.*; import com.sun.solaris.service.exception.*; /* * poold overview * ----- -------- * * poold manipulates system resources in accordance with administrator * specified constraints and objectives. The "goal" of the application * is to maximise the efficiency of available resources within these * parameters. * * Constraints are specified as follows: * * On a resource set: * * - min Is the minimum amount of resource a set should * receive. poold will never elect to move resource so that a set * falls below its minimum value. It is possible for a set to * fall below its minimum as a consequence of administrative * intervention, in which case poold will endeavour to bring a * set back to its minimum level at the earliest opportunity. * * - max Is the maximum amount of resource a set should * recieve. poold will never elect to move resource so that a set * rises above its maximum value. It is possible for a set to * rise above its maximum as a consequence of administrative * intervention, in which case poold will endeavour to bring a * set back to its maximum level at the earliest opportunity. * * On a resource component: * * - cpu.pinned Is an indication that a CPU should be ignored by * poold for purposes of reallocation. A pinned CPU will never be * moved by poold from one set to another. * * In addition to constraints, an administrator may also specify * objectives. Currently three types of objectives are supported: * * - system.wt-load Is an objective set across an entire * configuration. It attempts to ensure that resource is shared * in accordance with current consumption. Those partitions which * are most heavily utilized are give more resource in an attempt * to lower their utilization levels. * * - set.locality Is a locality objective which attempts to * minimize or maximize resource locality for a set. * * - set.utilization Is a utilization based objective which an * administrator may use to explicitly dictate the utilization * levels which should be achieved for a set. * * When executing, poold uses information about the current pools * configuration; monitored resource utilization and specified * constraints and objectives to determine if a move of resources is * likely to lead to an increase in objective satisfaction. * * Candidate moves are generated by observing resource constraints. * These moves are evaluated and scored in terms of their contribution * to objective satisfaction (note: objectives are weighted according * to their importance) and ranked accordingly. If a move can be * identified that delivers an improvement, the move is made. Data is * collected about past moves and recorded as "Decision History", * before the move is made this data is consulted and if the move is * expected not to yield an improvement, it may be cancelled. This * refinement is designed to improve the quality of decision making by * reflecting upon the past performance of decisions as well as the * contribution to objective satisfaction. * * poold structure * ----- --------- * * The common package name begins with: * * com.sun.solaris * * The software is divided into two main functional areas: * * service * * These packages collaborate to provide services to poold * (typically, they use JNI to access exising * functionality). They are not designed to be extended. For more * details on these classes examine the source files in each * directory. * * exception Stack trace formatter * kstat Interface to Solaris kstat facility * locality Interface to Solaris lgrp facility * logging Interface to Solaris syslog facility * pools Interface to Solaris libpool facility * timer High resolution timestamps * * domain: * * These package reflect problem domain concepts and are * responsible for application logic related to these * concepts. * * pools Dynamic Resource Pools specific functionality. * * This code block will continue to explain in more detail how poold * is organized. * * poold provides the following basic facilities: * * Monitoring: * * Basic statistic access is provided by the * com.sun.solaris.service.kstat package. The following interfaces and * classes collaborate to provide higher level statistic and * monitoring facilities to the application: * * INTERFACES * * AggregateStatistic * Monitor * Statistic * StatisticListener * * CLASSES * * AbstractStatistic * DoubleStatistic * LongStatistic * ResourceMonitor * StatisticEvent * StatisticList * StatisticOperations * SystemMonitor * UnsignedInt64Statistic * * Logging: * * Logging services are provided by the com.sun.solaris.logging * package. In addition, the following class implements Poold's * specific logging requirements. * * CLASSES * * Poold * * Optimization: * * lgrp service are provided by the com.sun.solaris.service.lgrp * package. pools services are provided by the * com.sun.solaris.service.pools package. In addition, optimization is * implemented in the following Interfaces and Classes: * * INTERFACES * * Objective * Solver * WorkloadDependentObjective * * CLASSES * * AbstractObjective * ComponentMove * DecisionHistory * Expression * KExpression * KVEpression * KVOpExpression * LocalityObjective * Move * Poold * QuantityMove * SystemSolver * UtilizationObjective * WeightedLoadObjective * * Configuration: * * pools services are provided by the com.sun.solaris.service.pools * package, this is used to read poold configuration details from a * libpool configuration. In addition, configuration is implemented in * the following Classes: * * CLASSES * * AbstractObjective * Expression * Poold * SystemSolver * * (NB: Some classes were mentioned in multiple categories where there * responsbilities overlap. Inner classes are not listed as their * responsibilities can be clearly inferred from their context.) * * For more details on any of the packages, classes or interfaces * mentioned above, look at the documentation associated with each * class. */ /** * The <code>Poold</code> class implements a dynamic resource * allocation system for Solaris. * * <code>Poold</code> is a monitoring daemon, designed to evaluate * user specified objectives, monitor workload behaviour and * dynamically assign resources in order to satisfy the evaluated * objectives. For more details see: * * <a href="http://sac.eng.sun.com/PSARC/2002/287/">PSARC/2002/287</a> */ final class Poold { /** * The configuration which is manipulated. */ private Configuration conf; /** * The monitoring interface. */ private Monitor monitor; /** * The interface to higher level resource managers. */ private DRM drm; /** * The interface to the configuration solver. */ private Solver solver; /** * Default path to the logging properties file. */ public static final String POOLD_PROPERTIES_PATH = "/usr/lib/pool/poold.properties"; /** * Logger for records which aren't produced in the Monitoring, * Configuration, or Optimization states. This logger is the * parent to the loggers used in those states. */ public static final Logger BASE_LOG = Logger.getLogger( "com.sun.solaris.domain.pools.poold"); /** * Logger for records produced in the Configuration state. */ public static final Logger CONF_LOG = Logger.getLogger( "com.sun.solaris.domain.pools.poold.Configuration"); /** * Logger for records produced in the Monitoring state. */ public static final Logger MON_LOG = Logger.getLogger( "com.sun.solaris.domain.pools.poold.Monitoring"); /** * Logger for records produced in the Optimization state. */ public static final Logger OPT_LOG = Logger.getLogger( "com.sun.solaris.domain.pools.poold.Optimization"); /** * Singleton instance of Poold. */ private static Poold instance; /** * The main sampling and solving thread. */ private Thread mainThread; /** * Process exit code indicating a failure. */ private static final int E_PO_FAILURE = 2; /** * Keep track of whether initialize() has been invoked, to * output the "starting" message on the first. */ private AtomicBoolean firstInitialization = new AtomicBoolean(true); /** * Flags whether poold should run or exit. */ private AtomicBoolean shouldRun = new AtomicBoolean(true); private static class logHelper { /** * Default logfile location */ public static final String DEF_LOG_LOC = "/var/log/pool/poold"; /** * Log location indicating <code>syslog</code>, as * opposed to a file, should be used. */ public static final String SYSLOG_LOG_LOC = "SYSLOG"; /** * Default Log severity (if not overridden) */ public static final Severity DEF_SEVERITY = Severity.INFO; /** * Name of configuration property, log location. */ public static final String PROPERTY_NAME_LOG_LOC = "system.poold.log-location"; /** * Name of configuration property, log level. */ public static final String PROPERTY_NAME_LOG_LEVEL = "system.poold.log-level"; /** * Location of logfile -- an absolute filename, or * "SYSLOG". */ private static String location; /** * Logfile handler, responsible for taking log messages * and exporting them. */ private static Handler handler; /** * Logfile severity, log messages below this severity are * ignored. */ private static Severity severity; /** * Flag recording whether preinitialization has occurred. */ private static boolean preinitialized = false; /** * Flag recording whether the logging Severity has been * overridden with the -l command-line option, which * means the console is the only thing being logged to, * and the configuration's logging properties are * ignored. */ private static boolean usingConsole; /** * Indicates whether logging semantics should be changed * to facilitate debugging. */ private static final boolean loggingDebugging = false; /** * Do the pre-initialization initialization: install * loggers for reporting errors during initialization. * * @param consoleSeverity If non-null, indicates that * the configuration property-controlled logging behavior * is to be overridden (the <code>-l</code> option was * specified), and messages are to be logged only to the * console, with (at most) the given maximum severity. */ private static void preinitialize(Severity consoleSeverity) { if (preinitialized) return; /* * Read logging properties affecting the * FileHandler and ConsoleHandler from * <code>poold.properties</code>. */ Properties props = new Properties(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); try { props.load( new FileInputStream(POOLD_PROPERTIES_PATH)); props.store(bos, ""); LogManager.getLogManager().readConfiguration( new ByteArrayInputStream( bos.toByteArray())); } catch (IOException ioe) { Poold.MON_LOG.log(Severity.WARNING, "could not " + "read logging properties from " + "poold.properties: " + ioe); } if (consoleSeverity == null || loggingDebugging) { /* * Log messages to /var/log/pool/poold, * the default location, until the pools * configuration properties are read and * applied, which may change the logging * file and severity. * * Under normal circumstances, it's * expected that NO INFO-level messages * will be emitted until that time; this * is only a measure to ensure that * unanticipated errors are reported in * some log. */ location = SYSLOG_LOG_LOC; handler = SyslogHandler.getInstance("poold", Facility.DAEMON); severity = DEF_SEVERITY; handler.setLevel(severity); BASE_LOG.addHandler(handler); } if (consoleSeverity != null) { /* * If -l is specified, log to the * console. Unless loggingDebug is * true, this will also mean that the * logging properties are ignored. * * Determine if the user has specified * the use of a ConsoleHandler through * poold.properties. */ Logger root = Logger.getLogger(""); Handler[] handler = root.getHandlers(); ConsoleHandler ch = null; for (int i = 0; i < handler.length && ch == null; i++) if (handler[i] instanceof ConsoleHandler) ch = (ConsoleHandler)handler[i]; /* * If none was previously, install a * ConsoleHandler. */ if (ch == null) { ch = new ConsoleHandler(); ch.setFormatter( new SysloglikeFormatter()); ch.setLevel(consoleSeverity); root.addHandler(ch); } severity = consoleSeverity; BASE_LOG.log(Severity.DEBUG, "logging with level " + severity); /** * Allow logging properties to be * effective if loggingDebugging is not * set. */ if (!loggingDebugging) usingConsole = true; } preinitialized = true; } /** * Configure loggers based on the logging-related * configuration properties. Outputs a description of * any changes to the configuration logger. * * @throws ConfigurationException if there is an error * applying libpool configuration properties to * <code>poold</code> */ public static void initializeWithConfiguration( Configuration conf) throws ConfigurationException { String newLogLocation; Severity newLogSeverity; String newLogSeverityName = null; /* * Set the log location as specified by the * configuration's system properties. */ try { newLogLocation = conf.getStringProperty( PROPERTY_NAME_LOG_LOC); } catch (PoolsException e) { newLogLocation = DEF_LOG_LOC; } try { newLogSeverityName = conf.getStringProperty( PROPERTY_NAME_LOG_LEVEL); newLogSeverity = Severity.getSeverityWithName( newLogSeverityName); assert(newLogSeverity != null); } catch (PoolsException e) { newLogSeverity = DEF_SEVERITY; } catch (IllegalArgumentException e) { throw(ConfigurationException) (new ConfigurationException( "invalid " + PROPERTY_NAME_LOG_LEVEL + "value: " + newLogSeverityName) .initCause(e)); } Handler newLogHandler = null; /* * (Re)install the logger for the poold class * hierarchy. This means that only poold * messages are controlled by the pools * configuration properties/command-line * options. */ /* * The logfile is always re-opened, in case the * cause for reinitialization is due to SIGHUP * following a log rotation. */ if (handler != null) { BASE_LOG.removeHandler(handler); handler.close(); handler = null; } if (newLogLocation.toUpperCase().equals( SYSLOG_LOG_LOC.toUpperCase())) newLogHandler = SyslogHandler.getInstance("poold", Facility.DAEMON); else { if (!newLogLocation.startsWith("/")) throw new ConfigurationException( PROPERTY_NAME_LOG_LOC + " value is not an" + " absolute path"); try { newLogHandler = new FileHandler( newLogLocation, 0, 1, true); newLogHandler.setFormatter( new SysloglikeFormatter()); } catch (java.io.IOException ioe) { Poold.utility.die( Poold.CONF_LOG, new PooldException( newLogLocation + ": can't write") .initCause(ioe), false); } } if (!severity.equals(newLogSeverity) || !location.equals(newLogLocation)) CONF_LOG.log(Severity.DEBUG, "logging with level " + severity); severity = newLogSeverity; handler = newLogHandler; location = newLogLocation; handler.setLevel(severity); BASE_LOG.addHandler(handler); } /** * Configure the loggers based on the pool's logging * properties, or, if the -l option was specified on the * command line, continue to use the console. */ public static void initialize(Configuration conf) throws ConfigurationException { if (usingConsole) return; else initializeWithConfiguration(conf); } /** * Return the current logging level. */ public static Severity getSeverity() { return (severity); } public static void close() { if (handler != null) { BASE_LOG.removeHandler(handler); handler.close(); } } } /** * Constructor * * Only one poold instance should be running per system. * * @param consoleSeverity If non-null, indicates that the * configuration property-controlled logging behavior is to be * overridden (the <code>-l</code> option was specified), and * messages are to be logged only to the console, with (at most) * the given maximum severity. */ private Poold(Severity consoleSeverity) { /* * Establish loggers for recording errors during * initialization. Under normal circumstances, no * messages will be emitted; this is only a measure to * make sure that unanticipated errors are reported in * some log, or the console, if the -l option is used. */ logHelper.preinitialize(consoleSeverity); /* * Try opening the configuration read-write in hopes the * ability will be possessed henceforth. */ try { conf = new Configuration(PoolInternal. pool_dynamic_location(), PoolInternal.PO_RDWR); conf.close(); } catch (PoolsException pe) { Poold.utility.die(CONF_LOG, new PooldException( "cannot open dynamic pools configuration " + "read-write (" + pe.getMessage() + ")") .initCause(pe), false); } try { conf = new Configuration(PoolInternal. pool_dynamic_location(), PoolInternal.PO_RDONLY); } catch (PoolsException pe) { Poold.utility.die(CONF_LOG, pe); } /* * Create the required sub-components: * - a monitoring object * - a DRM implementer * - a solver */ monitor = new SystemMonitor(); drm = new LogDRM(); solver = new SystemSolver(monitor); } /** * Returns a reference to the singleton <code>Poold</code>, * constructing one if necessary. * * @param consoleSeverity If non-null, indicates that the * configuration property-controlled logging behavior is to be * overridden (the <code>-l</code> option was specified), and * messages are to be logged only to the console, with (at most) * the given maximum severity. * @throws IllegalArgumentException if the given console * severity doesn't match that of an existing instance. */ public static Poold getInstanceWithConsoleLogging( Severity consoleSeverity) { if (instance == null) return (instance = new Poold(consoleSeverity)); else if (logHelper.usingConsole == false && consoleSeverity == null || consoleSeverity != null && logHelper.getSeverity().equals( consoleSeverity)) return (instance); else throw new IllegalArgumentException(); } /** * Initializes <code>Poold</code> for operation at startup or * in response to a detected libpool configuration change. */ private void initialize() { try { logHelper.initialize(conf); if (firstInitialization.get()) CONF_LOG.log(Severity.INFO, "starting"); /* * When a system is extremely busy, it may * prove difficult to initialize poold. Just * keep trying until we succeed. */ boolean busy = true; while (busy && shouldRun.get()) { busy = false; try { monitor.initialize(conf); CONF_LOG.log(Severity.DEBUG, "configuring solver..."); solver.initialize(conf); CONF_LOG.log(Severity.INFO, "configuration complete"); } catch (PoolsException pe) { CONF_LOG.log(Severity.INFO, "The system is too busy to " + "initialize, attempting " + "initialization again"); /* * pause for a while before * re-attempting the * initialization. */ try { Thread.sleep(50); } catch (InterruptedException ie) { /* * Safe to ignore this * exception as we * will simply try * again sooner. */ } busy = true; } catch (StaleMonitorException sme) { CONF_LOG.log(Severity.INFO, "The system is too busy to " + "initialize, attempting " + "initialization again"); /* * pause for a while before * re-attempting the * initialization. */ try { Thread.sleep(50); } catch (InterruptedException ie) { /* * Safe to ignore this * exception as we * will simply try * again sooner. */ } busy = true; } } if (firstInitialization.get()) firstInitialization.set(false); } catch (ConfigurationException ce) { Poold.utility.die(CONF_LOG, ce); } } /** * Execute <code>Poold</code> indefinitely. This method is * invoked after <code>Poold</code> completes * configuration. It will continue to execute until * <code>Poold</code> is terminated. * * @throws Exception If an there is an error in execution. */ private void execute() throws Exception { int changed = 0; boolean confRequired = false; while (shouldRun.get()) { try { changed = conf.update(); assert(!confRequired || confRequired && changed != 0); if (changed != 0) { CONF_LOG.log(Severity.DEBUG, "configuration change detected"); if (!confRequired) CONF_LOG.log(Severity.INFO, "configuration changed " + "externally"); CONF_LOG.log(Severity.INFO, "reconfiguring..."); } confRequired = false; } catch (PoolsException pe) { Poold.utility.die(CONF_LOG, pe); } if (changed != 0) initialize(); boolean gotNext = false; while (shouldRun.get() && !gotNext) { try { monitor.getNext(); gotNext = true; /* * All workload-dependent * objectives must now be * checked for violations. The * solver holds all objectives * and it makes the decision * about whether a * reconfiguration is required. */ if (solver.examine(monitor)) { MON_LOG.log(Severity.INFO, "reconfiguration required"); confRequired = solver.solve(); } else { MON_LOG.log(Severity.INFO, "all evaluated objectives" + " satisfied"); } } catch (StaleMonitorException e) { /* * Later, assert that every * cause of the * StaleMonitorException is * handled by the above * conf.update(). */ confRequired = true; } catch (InterruptedException ie) { Poold.MON_LOG.log(Severity.INFO, "interrupted"); break; } } if (!shouldRun.get()) break; System.runFinalization(); } Poold.BASE_LOG.log(Severity.NOTICE, "exiting"); } /** * Cleanup any resources when the application terminates. */ private void cleanup() { conf.close(); logHelper.close(); instance = null; } /** * Invoke <code>Poold</code>. This main function is provided * so that <code>poold</code> can be executed. Execution will * continue indefinitely unless there is an error, in which case * when execute() terminates. */ public void run() { mainThread = Thread.currentThread(); Runtime.getRuntime().addShutdownHook(new Thread() { public void run() { try { cleanup(); } catch (Throwable t) { } } }); try { initialize(); execute(); } catch (Throwable t) { Poold.utility.die(BASE_LOG, t); } } /** * Stops <code>Poold</code>. Sets a flag indicating that run() * should break out of the monitor/solve loop and clean up. */ public void shutdown() { /* * Flag that the main thread should break out of the * sample/solve loop as soon as possible. */ shouldRun.set(false); /* * Interrupt the main thread, which will cause the * monitor to break out of its sleep if it's waiting for * the sample time to arrive, and cause the sample/solve * loop condition to be evaluated sooner. But make some * effort not to cause an InterruptedIOException if * we're not even through initialization yet; we'll get * around to shutting down soon enough. */ if (!firstInitialization.get() && mainThread != null) mainThread.interrupt(); } public static void main(String args[]) throws IllegalArgumentException { Severity severity = null; if (args.length > 0) { if (args[0].compareTo("-l") == 0 && args.length == 2) { severity = (Severity) Severity.parse(args[1]); } else throw new IllegalArgumentException( "usage: poold [-l level]"); } Poold p = getInstanceWithConsoleLogging(severity); p.run(); } /** * The <code>utility</code> class provides various * <code>Poold</code> related static utility methods that * don't naturally reside on any class. */ static class utility { /** * Outputs a near-final message corresponding * to an exception (or other throwable) to the named * logger before causing the VM to exit. The message * will have ERROR severity unless an instance of * PoolsException is given, in which case the message * will adopt the PoolsException's severity. Similarly, * if the PoolsException specifies an exit code, it will * be used; otherwise the default of * <code>E_PO_FAILURE</code> will be used. * * @param logger Logger used to log the message * @param t The cause. A stack trace will be affixed to * the message. */ public static void die(Logger logger, Throwable t) { die(logger, t, true); } /** * Outputs a near-final message corresponding * to an exception (or other throwable) to the named * logger before causing the VM to exit. The message * will have ERROR severity unless an instance of * PoolsException is given, in which case the message * will adopt the PoolsException's severity. Similarly, * if the PoolsException specifies an exit code, it will * be used; otherwise the default of * <code>E_PO_FAILURE</code> will be used. * * @param logger Logger used to log the message * @param t The cause. * @param showStackTrace If true, a stack trace will be * affixed to the message. */ public static void die(Logger logger, Throwable t, boolean showStackTrace) { try { Severity severity; /* * Configure the message's exception and * severity. */ LogRecord record; if (t instanceof PooldException) record = new LogRecord( ((PooldException)t).getSeverity(), t.getMessage()); else record = new LogRecord(Severity.ERR, t.getMessage()); if (record.getMessage() == null) record.setMessage("exception " + t.getClass().getName()); if (showStackTrace) record.setThrown(t); record.setLoggerName(logger.getName()); logger.log(record); if (logHelper.handler != null) logHelper.handler.flush(); if (t instanceof PooldException) System.exit(((PooldException)t) .getExitCode()); else System.exit(E_PO_FAILURE); } catch (Exception e) { SuccinctStackTraceFormatter.printStackTrace(e); System.exit(-1); } } /** * Outputs a warning-level message to the named logger. * * @param logger Logger used to log the message * @param t The cause. * @param showStackTrace If true, a stack trace will be * affixed to the message. */ public static void warn(Logger logger, Throwable t, boolean showStackTrace) { try { Severity severity; /* * Configure the message's exception and * severity. */ LogRecord record; if (t instanceof PooldException) record = new LogRecord( ((PooldException)t).getSeverity(), t.getMessage()); else record = new LogRecord(Severity.WARNING, t.getMessage()); if (record.getMessage() == null) record.setMessage("exception " + t.getClass().getName()); if (showStackTrace) record.setThrown(t); record.setLoggerName(logger.getName()); logger.log(record); if (logHelper.handler != null) logHelper.handler.flush(); } catch (Exception e) { SuccinctStackTraceFormatter.printStackTrace(e); System.exit(-1); } } } } class ConfigurationException extends Exception { public ConfigurationException(String message) { super(message); } } class IllegalOFValueException extends RuntimeException { public IllegalOFValueException(String message) { super(message); } } class PooldException extends Exception { /** * The exit code which the virtual machine should exit at if * this exception cannot be handled. */ private int exitCode; /** * The severity of this message. See <code>Severity</code>. */ private Severity severity; /** * Constructs a message with default exit code and * <code>System.ERR</code> severity. */ public PooldException(String message) { this(message, 1, Severity.ERR); } /** * Constructs a message with given exit code and * <code>System.ERR</code> severity. */ public PooldException(String message, int exitCode) { this(message, exitCode, Severity.ERR); } /** * Constructs a message with a given exit code and severity. */ public PooldException(String message, int exitCode, Severity severity) { super(message); this.exitCode = exitCode; this.severity = severity; } /** * The exit code which the virtual machine should exit at if * this exception cannot be handled. */ public int getExitCode() { return (exitCode); } public Severity getSeverity() { return (severity); } }
