/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright 2012 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014, 2016 by Delphix. All rights reserved. * Copyright 2020 Joyent, Inc. */ #include <sys/types.h> #include <sys/param.h> #include <sys/systm.h> #include <sys/disp.h> #include <sys/var.h> #include <sys/cmn_err.h> #include <sys/debug.h> #include <sys/x86_archext.h> #include <sys/archsystm.h> #include <sys/cpuvar.h> #include <sys/psm_defs.h> #include <sys/clock.h> #include <sys/atomic.h> #include <sys/lockstat.h> #include <sys/smp_impldefs.h> #include <sys/dtrace.h> #include <sys/time.h> #include <sys/panic.h> #include <sys/cpu.h> #include <sys/sdt.h> #include <sys/comm_page.h> #include <sys/bootconf.h> #include <sys/kobj.h> #include <sys/kobj_lex.h> #include <sys/tsc.h> #include <sys/prom_debug.h> #include <util/qsort.h> /* * Using the Pentium's TSC register for gethrtime() * ------------------------------------------------ * * The Pentium family, like many chip architectures, has a high-resolution * timestamp counter ("TSC") which increments once per CPU cycle. The contents * of the timestamp counter are read with the RDTSC instruction. * * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count * must be translated into nanoseconds in order to implement gethrtime(). * We avoid inducing floating point operations in this conversion by * implementing the same nsec_scale algorithm as that found in the sun4u * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains * a detailed description of the algorithm; the comment is not reproduced * here. This implementation differs only in its value for NSEC_SHIFT: * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for * 60 MHz Pentiums. * * While TSC and %tick are both cycle counting registers, TSC's functionality * falls short in several critical ways: * * (a) TSCs on different CPUs are not guaranteed to be in sync. While in * practice they often _are_ in sync, this isn't guaranteed by the * architecture. * * (b) The TSC cannot be reliably set to an arbitrary value. The architecture * only supports writing the low 32-bits of TSC, making it impractical * to rewrite. * * (c) The architecture doesn't have the capacity to interrupt based on * arbitrary values of TSC; there is no TICK_CMPR equivalent. * * Together, (a) and (b) imply that software must track the skew between * TSCs and account for it (it is assumed that while there may exist skew, * there does not exist drift). To determine the skew between CPUs, we * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing * the online operation calls tsc_sync_master(). * * In the absence of time-of-day clock adjustments, gethrtime() must stay in * sync with gettimeofday(). This is problematic; given (c), the software * cannot drive its time-of-day source from TSC, and yet they must somehow be * kept in sync. We implement this by having a routine, tsc_tick(), which * is called once per second from the interrupt which drives time-of-day. * * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified * atomically with nsec_scale under CLOCK_LOCK. This assures that time * monotonically increases. */ #define NSEC_SHIFT 5 static uint_t nsec_unscale; /* * These two variables used to be grouped together inside of a structure that * lived on a single cache line. A regression (bug ID 4623398) caused the * compiler to emit code that "optimized" away the while-loops below. The * result was that no synchronization between the onlining and onlined CPUs * took place. */ static volatile int tsc_ready; static volatile int tsc_sync_go; /* * Used as indices into the tsc_sync_snaps[] array. */ #define TSC_MASTER 0 #define TSC_SLAVE 1 /* * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. */ #define TSC_SYNC_STOP 1 #define TSC_SYNC_GO 2 #define TSC_SYNC_DONE 3 #define SYNC_ITERATIONS 10 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ unsigned int *_l = (unsigned int *)&(tsc); \ (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ } #define TSC_CONVERT(tsc, hrt, scale) { \ unsigned int *_l = (unsigned int *)&(tsc); \ (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ } int tsc_master_slave_sync_needed = 1; typedef struct tsc_sync { volatile hrtime_t master_tsc, slave_tsc; } tsc_sync_t; static tsc_sync_t *tscp; static hrtime_t tsc_last_jumped = 0; static int tsc_jumped = 0; static uint32_t tsc_wayback = 0; /* * The cap of 1 second was chosen since it is the frequency at which the * tsc_tick() function runs which means that when gethrtime() is called it * should never be more than 1 second since tsc_last was updated. */ static hrtime_t tsc_resume_cap_ns = NANOSEC; /* 1s */ static hrtime_t shadow_tsc_hrtime_base; static hrtime_t shadow_tsc_last; static uint_t shadow_nsec_scale; static uint32_t shadow_hres_lock; int get_tsc_ready(); /* * Allow an operator specify an explicit TSC calibration source * via /etc/system e.g. `set tsc_calibration="pit"` */ char *tsc_calibration; /* * The source that was used to calibrate the TSC. This is currently just * for diagnostic purposes. */ static tsc_calibrate_t *tsc_calibration_source; /* The TSC frequency after calibration */ static uint64_t tsc_freq; static inline hrtime_t tsc_protect(hrtime_t a) { if (a > tsc_resume_cap) { atomic_inc_32(&tsc_wayback); DTRACE_PROBE3(tsc__wayback, htrime_t, a, hrtime_t, tsc_last, uint32_t, tsc_wayback); return (tsc_resume_cap); } return (a); } hrtime_t tsc_gethrtime(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; do { old_hres_lock = hres_lock; if ((tsc = tsc_read()) >= tsc_last) { /* * It would seem to be obvious that this is true * (that is, the past is less than the present), * but it isn't true in the presence of suspend/resume * cycles. If we manage to call gethrtime() * after a resume, but before the first call to * tsc_tick(), we will see the jump. In this case, * we will simply use the value in TSC as the delta. */ tsc -= tsc_last; } else if (tsc >= tsc_last - 2*tsc_max_delta) { /* * There is a chance that tsc_tick() has just run on * another CPU, and we have drifted just enough so that * we appear behind tsc_last. In this case, force the * delta to be zero. */ tsc = 0; } else { /* * If we reach this else clause we assume that we have * gone through a suspend/resume cycle and use the * current tsc value as the delta. * * In rare cases we can reach this else clause due to * a lack of monotonicity in the TSC value. In such * cases using the current TSC value as the delta would * cause us to return a value ~2x of what it should * be. To protect against these cases we cap the * suspend/resume delta at tsc_resume_cap. */ tsc = tsc_protect(tsc); } hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); } while ((old_hres_lock & ~1) != hres_lock); return (hrt); } hrtime_t tsc_gethrtime_delta(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; ulong_t flags; do { old_hres_lock = hres_lock; /* * We need to disable interrupts here to assure that we * don't migrate between the call to tsc_read() and * adding the CPU's TSC tick delta. Note that disabling * and reenabling preemption is forbidden here because * we may be in the middle of a fast trap. In the amd64 * kernel we cannot tolerate preemption during a fast * trap. See _update_sregs(). */ flags = clear_int_flag(); tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* See comments in tsc_gethrtime() above */ if (tsc >= tsc_last) { tsc -= tsc_last; } else if (tsc >= tsc_last - 2 * tsc_max_delta) { tsc = 0; } else { tsc = tsc_protect(tsc); } hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); } while ((old_hres_lock & ~1) != hres_lock); return (hrt); } hrtime_t tsc_gethrtime_tick_delta(void) { hrtime_t hrt; ulong_t flags; flags = clear_int_flag(); hrt = tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); return (hrt); } /* Calculate the hrtime while exposing the parameters of that calculation. */ hrtime_t tsc_gethrtime_params(uint64_t *tscp, uint32_t *scalep, uint8_t *shiftp) { uint32_t old_hres_lock, scale; hrtime_t tsc, last, base; do { old_hres_lock = hres_lock; if (gethrtimef == tsc_gethrtime_delta) { ulong_t flags; flags = clear_int_flag(); tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); } else { tsc = tsc_read(); } last = tsc_last; base = tsc_hrtime_base; scale = nsec_scale; } while ((old_hres_lock & ~1) != hres_lock); /* See comments in tsc_gethrtime() above */ if (tsc >= last) { tsc -= last; } else if (tsc >= last - 2 * tsc_max_delta) { tsc = 0; } else { tsc = tsc_protect(tsc); } TSC_CONVERT_AND_ADD(tsc, base, nsec_scale); if (tscp != NULL) { /* * Do not simply communicate the delta applied to the hrtime * base, but rather the effective TSC measurement. */ *tscp = tsc + last; } if (scalep != NULL) { *scalep = scale; } if (shiftp != NULL) { *shiftp = NSEC_SHIFT; } return (base); } /* * This is similar to tsc_gethrtime_delta, but it cannot actually spin on * hres_lock. As a result, it caches all of the variables it needs; if the * variables don't change, it's done. */ hrtime_t dtrace_gethrtime(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; ulong_t flags; do { old_hres_lock = hres_lock; /* * Interrupts are disabled to ensure that the thread isn't * migrated between the tsc_read() and adding the CPU's * TSC tick delta. */ flags = clear_int_flag(); tsc = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) tsc += tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* * See the comments in tsc_gethrtime(), above. */ if (tsc >= tsc_last) tsc -= tsc_last; else if (tsc >= tsc_last - 2*tsc_max_delta) tsc = 0; else tsc = tsc_protect(tsc); hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); if ((old_hres_lock & ~1) == hres_lock) break; /* * If we're here, the clock lock is locked -- or it has been * unlocked and locked since we looked. This may be due to * tsc_tick() running on another CPU -- or it may be because * some code path has ended up in dtrace_probe() with * CLOCK_LOCK held. We'll try to determine that we're in * the former case by taking another lap if the lock has * changed since when we first looked at it. */ if (old_hres_lock != hres_lock) continue; /* * So the lock was and is locked. We'll use the old data * instead. */ old_hres_lock = shadow_hres_lock; /* * Again, disable interrupts to ensure that the thread * isn't migrated between the tsc_read() and adding * the CPU's TSC tick delta. */ flags = clear_int_flag(); tsc = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) tsc += tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* * See the comments in tsc_gethrtime(), above. */ if (tsc >= shadow_tsc_last) tsc -= shadow_tsc_last; else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) tsc = 0; else tsc = tsc_protect(tsc); hrt = shadow_tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); } while ((old_hres_lock & ~1) != shadow_hres_lock); return (hrt); } hrtime_t tsc_gethrtimeunscaled(void) { uint32_t old_hres_lock; hrtime_t tsc; do { old_hres_lock = hres_lock; /* See tsc_tick(). */ tsc = tsc_read() + tsc_last_jumped; } while ((old_hres_lock & ~1) != hres_lock); return (tsc); } /* * Convert a nanosecond based timestamp to tsc */ uint64_t tsc_unscalehrtime(hrtime_t nsec) { hrtime_t tsc; if (tsc_gethrtime_enable) { TSC_CONVERT(nsec, tsc, nsec_unscale); return (tsc); } return ((uint64_t)nsec); } /* Convert a tsc timestamp to nanoseconds */ void tsc_scalehrtime(hrtime_t *tsc) { hrtime_t hrt; hrtime_t mytsc; if (tsc == NULL) return; mytsc = *tsc; TSC_CONVERT(mytsc, hrt, nsec_scale); *tsc = hrt; } hrtime_t tsc_gethrtimeunscaled_delta(void) { hrtime_t hrt; ulong_t flags; /* * Similarly to tsc_gethrtime_delta, we need to disable preemption * to prevent migration between the call to tsc_gethrtimeunscaled * and adding the CPU's hrtime delta. Note that disabling and * reenabling preemption is forbidden here because we may be in the * middle of a fast trap. In the amd64 kernel we cannot tolerate * preemption during a fast trap. See _update_sregs(). */ flags = clear_int_flag(); hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); return (hrt); } /* * TSC Sync Master * * Typically called on the boot CPU, this attempts to quantify TSC skew between * different CPUs. If an appreciable difference is found, gethrtimef will be * changed to point to tsc_gethrtime_delta(). * * Calculating skews is precise only when the master and slave TSCs are read * simultaneously; however, there is no algorithm that can read both CPUs in * perfect simultaneity. The proposed algorithm is an approximate method based * on the behaviour of cache management. The slave CPU continuously polls the * TSC while reading a global variable updated by the master CPU. The latest * TSC reading is saved when the master's update (forced via mfence) reaches * visibility on the slave. The master will also take a TSC reading * immediately following the mfence. * * While the delay between cache line invalidation on the slave and mfence * completion on the master is not repeatable, the error is heuristically * assumed to be 1/4th of the write time recorded by the master. Multiple * samples are taken to control for the variance caused by external factors * such as bus contention. Each sample set is independent per-CPU to control * for differing memory latency on NUMA systems. * * TSC sync is disabled in the context of virtualization because the CPUs * assigned to the guest are virtual CPUs which means the real CPUs on which * guest runs keep changing during life time of guest OS. So we would end up * calculating TSC skews for a set of CPUs during boot whereas the guest * might migrate to a different set of physical CPUs at a later point of * time. */ void tsc_sync_master(processorid_t slave) { ulong_t flags, source, min_write_time = ~0UL; hrtime_t write_time, mtsc_after, last_delta = 0; tsc_sync_t *tsc = tscp; int cnt; int hwtype; hwtype = get_hwenv(); if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) return; flags = clear_int_flag(); source = CPU->cpu_id; for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { while (tsc_sync_go != TSC_SYNC_GO) SMT_PAUSE(); tsc->master_tsc = tsc_read(); membar_enter(); mtsc_after = tsc_read(); while (tsc_sync_go != TSC_SYNC_DONE) SMT_PAUSE(); write_time = mtsc_after - tsc->master_tsc; if (write_time <= min_write_time) { hrtime_t tdelta; tdelta = tsc->slave_tsc - mtsc_after; if (tdelta < 0) tdelta = -tdelta; /* * If the margin exists, subtract 1/4th of the measured * write time from the master's TSC value. This is an * estimate of how late the mfence completion came * after the slave noticed the cache line change. */ if (tdelta > (write_time/4)) { tdelta = tsc->slave_tsc - (mtsc_after - (write_time/4)); } else { tdelta = tsc->slave_tsc - mtsc_after; } last_delta = tsc_sync_tick_delta[source] - tdelta; tsc_sync_tick_delta[slave] = last_delta; min_write_time = write_time; } tsc->master_tsc = tsc->slave_tsc = write_time = 0; membar_enter(); tsc_sync_go = TSC_SYNC_STOP; } /* * Only enable the delta variants of the TSC functions if the measured * skew is greater than the fastest write time. */ last_delta = (last_delta < 0) ? -last_delta : last_delta; if (last_delta > min_write_time) { gethrtimef = tsc_gethrtime_delta; gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; tsc_ncpu = NCPU; } restore_int_flag(flags); } /* * TSC Sync Slave * * Called by a CPU which has just been onlined. It is expected that the CPU * performing the online operation will call tsc_sync_master(). * * Like tsc_sync_master, this logic is skipped on virtualized platforms. */ void tsc_sync_slave(void) { ulong_t flags; hrtime_t s1; tsc_sync_t *tsc = tscp; int cnt; int hwtype; hwtype = get_hwenv(); if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) return; flags = clear_int_flag(); for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { /* Re-fill the cache line */ s1 = tsc->master_tsc; membar_enter(); tsc_sync_go = TSC_SYNC_GO; do { /* * Do not put an SMT_PAUSE here. If the master and * slave are the same hyper-threaded CPU, we want the * master to yield as quickly as possible to the slave. */ s1 = tsc_read(); } while (tsc->master_tsc == 0); tsc->slave_tsc = s1; membar_enter(); tsc_sync_go = TSC_SYNC_DONE; while (tsc_sync_go != TSC_SYNC_STOP) SMT_PAUSE(); } restore_int_flag(flags); } /* * Called once per second on a CPU from the cyclic subsystem's * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) */ void tsc_tick(void) { hrtime_t now, delta; ushort_t spl; /* * Before we set the new variables, we set the shadow values. This * allows for lock free operation in dtrace_gethrtime(). */ lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, ipltospl(CBE_HIGH_PIL), &spl); shadow_tsc_hrtime_base = tsc_hrtime_base; shadow_tsc_last = tsc_last; shadow_nsec_scale = nsec_scale; shadow_hres_lock++; splx(spl); CLOCK_LOCK(&spl); now = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) now += tsc_sync_tick_delta[CPU->cpu_id]; if (now < tsc_last) { /* * The TSC has just jumped into the past. We assume that * this is due to a suspend/resume cycle, and we're going * to use the _current_ value of TSC as the delta. This * will keep tsc_hrtime_base correct. We're also going to * assume that rate of tsc does not change after a suspend * resume (i.e nsec_scale remains the same). */ delta = now; delta = tsc_protect(delta); tsc_last_jumped += tsc_last; tsc_jumped = 1; } else { /* * Determine the number of TSC ticks since the last clock * tick, and add that to the hrtime base. */ delta = now - tsc_last; } TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); tsc_last = now; CLOCK_UNLOCK(spl); } void tsc_hrtimeinit(uint64_t cpu_freq_hz) { extern int gethrtime_hires; longlong_t tsc; ulong_t flags; /* * cpu_freq_hz is the measured cpu frequency in hertz */ /* * We can't accommodate CPUs slower than 31.25 MHz. */ ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); nsec_scale = (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); nsec_unscale = (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC); flags = clear_int_flag(); tsc = tsc_read(); (void) tsc_gethrtime(); tsc_max_delta = tsc_read() - tsc; restore_int_flag(flags); gethrtimef = tsc_gethrtime; gethrtimeunscaledf = tsc_gethrtimeunscaled; scalehrtimef = tsc_scalehrtime; unscalehrtimef = tsc_unscalehrtime; hrtime_tick = tsc_tick; gethrtime_hires = 1; /* * Being part of the comm page, tsc_ncpu communicates the published * length of the tsc_sync_tick_delta array. This is kept zeroed to * ignore the absent delta data while the TSCs are synced. */ tsc_ncpu = 0; /* * Allocate memory for the structure used in the tsc sync logic. * This structure should be aligned on a multiple of cache line size. */ tscp = kmem_zalloc(PAGESIZE, KM_SLEEP); /* * Convert the TSC resume cap ns value into its unscaled TSC value. * See tsc_gethrtime(). */ if (tsc_resume_cap == 0) TSC_CONVERT(tsc_resume_cap_ns, tsc_resume_cap, nsec_unscale); } int get_tsc_ready() { return (tsc_ready); } /* * Adjust all the deltas by adding the passed value to the array and activate * the "delta" versions of the gethrtime functions. It is possible that the * adjustment could be negative. Such may occur if the SunOS instance was * moved by a virtual manager to a machine with a higher value of TSC. */ void tsc_adjust_delta(hrtime_t tdelta) { int i; for (i = 0; i < NCPU; i++) { tsc_sync_tick_delta[i] += tdelta; } gethrtimef = tsc_gethrtime_delta; gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; tsc_ncpu = NCPU; } /* * Functions to manage TSC and high-res time on suspend and resume. */ /* tod_ops from "uts/i86pc/io/todpc_subr.c" */ extern tod_ops_t *tod_ops; static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ static timestruc_t tsc_saved_ts; static int tsc_needs_resume = 0; /* We only want to do this once. */ int tsc_delta_onsuspend = 0; int tsc_adjust_seconds = 1; int tsc_suspend_count = 0; int tsc_resume_in_cyclic = 0; /* * Take snapshots of the current time and do any other pre-suspend work. */ void tsc_suspend(void) { /* * We need to collect the time at which we suspended here so we know * now much should be added during the resume. This is called by each * CPU, so reentry must be properly handled. */ if (tsc_gethrtime_enable) { /* * Perform the tsc_read after acquiring the lock to make it as * accurate as possible in the face of contention. */ mutex_enter(&tod_lock); tsc_saved_tsc = tsc_read(); tsc_saved_ts = TODOP_GET(tod_ops); mutex_exit(&tod_lock); /* We only want to do this once. */ if (tsc_needs_resume == 0) { if (tsc_delta_onsuspend) { tsc_adjust_delta(tsc_saved_tsc); } else { tsc_adjust_delta(nsec_scale); } tsc_suspend_count++; } } invalidate_cache(); tsc_needs_resume = 1; } /* * Restore all timestamp state based on the snapshots taken at suspend time. */ void tsc_resume(void) { /* * We only need to (and want to) do this once. So let the first * caller handle this (we are locked by the cpu lock), as it * is preferential that we get the earliest sync. */ if (tsc_needs_resume) { /* * If using the TSC, adjust the delta based on how long * we were sleeping (or away). We also adjust for * migration and a grown TSC. */ if (tsc_saved_tsc != 0) { timestruc_t ts; hrtime_t now, sleep_tsc = 0; int sleep_sec; extern void tsc_tick(void); extern uint64_t cpu_freq_hz; /* tsc_read() MUST be before TODOP_GET() */ mutex_enter(&tod_lock); now = tsc_read(); ts = TODOP_GET(tod_ops); mutex_exit(&tod_lock); /* Compute seconds of sleep time */ sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; /* * If the saved sec is less that or equal to * the current ts, then there is likely a * problem with the clock. Assume at least * one second has passed, so that time goes forward. */ if (sleep_sec <= 0) { sleep_sec = 1; } /* How many TSC's should have occured while sleeping */ if (tsc_adjust_seconds) sleep_tsc = sleep_sec * cpu_freq_hz; /* * We also want to subtract from the "sleep_tsc" * the current value of tsc_read(), so that our * adjustment accounts for the amount of time we * have been resumed _or_ an adjustment based on * the fact that we didn't actually power off the * CPU (migration is another issue, but _should_ * also comply with this calculation). If the CPU * never powered off, then: * 'now == sleep_tsc + saved_tsc' * and the delta will effectively be "0". */ sleep_tsc -= now; if (tsc_delta_onsuspend) { tsc_adjust_delta(sleep_tsc); } else { tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); } tsc_saved_tsc = 0; tsc_tick(); } tsc_needs_resume = 0; } } static int tsc_calibrate_cmp(const void *a, const void *b) { const tsc_calibrate_t * const *a1 = a; const tsc_calibrate_t * const *b1 = b; const tsc_calibrate_t *l = *a1; const tsc_calibrate_t *r = *b1; /* Sort from highest preference to lowest preference */ if (l->tscc_preference > r->tscc_preference) return (-1); if (l->tscc_preference < r->tscc_preference) return (1); /* For equal preference sources, sort alphabetically */ int c = strcmp(l->tscc_source, r->tscc_source); if (c < 0) return (-1); if (c > 0) return (1); return (0); } SET_DECLARE(tsc_calibration_set, tsc_calibrate_t); static tsc_calibrate_t * tsc_calibrate_get_force(const char *source) { tsc_calibrate_t **tsccpp; VERIFY3P(source, !=, NULL); SET_FOREACH(tsccpp, tsc_calibration_set) { tsc_calibrate_t *tsccp = *tsccpp; if (strcasecmp(source, tsccp->tscc_source) == 0) return (tsccp); } /* * If an operator explicitly gave a TSC value and we didn't find it, * we should let them know. */ cmn_err(CE_NOTE, "Explicit TSC calibration source '%s' not found; using default", source); return (NULL); } /* * As described in tscc_pit.c, as an intertim measure as we transition to * alternate calibration sources besides the PIT, we still want to gather * what the values would have been had we used the PIT. Therefore, if we're * using a source other than the PIT, we explicitly run the PIT calibration * which will store the TSC frequency as measured by the PIT for the * benefit of the APIC code (as well as any potential diagnostics). */ static void tsc_pit_also(void) { tsc_calibrate_t *pit = tsc_calibrate_get_force("PIT"); uint64_t dummy; /* We should always have the PIT as a possible calibration source */ VERIFY3P(pit, !=, NULL); /* If we used the PIT to calibrate, we don't need to run again */ if (tsc_calibration_source == pit) return; /* * Since we're not using the PIT as the actual TSC calibration source, * we don't care about the results or saving the result -- tscc_pit.c * saves the frequency in a global for the benefit of the APIC code. */ (void) pit->tscc_calibrate(&dummy); } uint64_t tsc_calibrate(void) { tsc_calibrate_t **tsccpp, *force; size_t tsc_set_size; int tsc_name_len; /* * Every x86 system since the Pentium has TSC support. Since we * only support 64-bit x86 systems, there should always be a TSC * present, and something's horribly wrong if it's missing. */ if (!is_x86_feature(x86_featureset, X86FSET_TSC)) panic("System does not have TSC support"); /* * If we already successfully calibrated the TSC, no need to do * it again. */ if (tsc_freq > 0) return (tsc_freq); PRM_POINT("Calibrating the TSC..."); /* * Allow an operator to explicitly specify a calibration source via * `set tsc_calibration=foo` in the bootloader or * `set tsc_calibration="foo"` in /etc/system (preferring a bootloader * supplied value over /etc/system). * * If no source is given, or the specified source is not found, we * fallback to trying all of the known sources in order by preference * (high preference value to low preference value) until one succeeds. */ tsc_name_len = BOP_GETPROPLEN(bootops, "tsc_calibration"); if (tsc_name_len > 0) { /* Overwrite any /etc/system supplied value */ if (tsc_calibration != NULL) { size_t len = strlen(tsc_calibration) + 1; kobj_free_string(tsc_calibration, len); } tsc_calibration = kmem_zalloc(tsc_name_len + 1, KM_SLEEP); BOP_GETPROP(bootops, "tsc_calibration", tsc_calibration); } if (tsc_calibration != NULL && (force = tsc_calibrate_get_force(tsc_calibration)) != NULL) { if (tsc_name_len > 0) { PRM_POINT("Forcing bootloader specified TSC calibration" " source"); } else { PRM_POINT("Forcing /etc/system specified TSC " "calibration source"); } PRM_DEBUGS(force->tscc_source); if (!force->tscc_calibrate(&tsc_freq)) panic("Failed to calibrate the TSC"); tsc_calibration_source = force; /* * We've saved the tsc_calibration_t that matched the value * of tsc_calibration at this point, so we can release the * memory for the value now. */ if (tsc_name_len > 0) { kmem_free(tsc_calibration, tsc_name_len + 1); } else if (tsc_calibration != NULL) { size_t len = strlen(tsc_calibration) + 1; kobj_free_string(tsc_calibration, len); } tsc_calibration = NULL; tsc_pit_also(); return (tsc_freq); } /* * While we could sort the set contents in place, we'll make a copy * of the set and avoid modifying the original set. */ tsc_set_size = SET_COUNT(tsc_calibration_set) * sizeof (tsc_calibrate_t **); tsccpp = kmem_zalloc(tsc_set_size, KM_SLEEP); bcopy(SET_BEGIN(tsc_calibration_set), tsccpp, tsc_set_size); /* * Sort by preference, highest to lowest */ qsort(tsccpp, SET_COUNT(tsc_calibration_set), sizeof (tsc_calibrate_t **), tsc_calibrate_cmp); for (uint_t i = 0; i < SET_COUNT(tsc_calibration_set); i++) { PRM_DEBUGS(tsccpp[i]->tscc_source); if (tsccpp[i]->tscc_calibrate(&tsc_freq)) { VERIFY3U(tsc_freq, >, 0); cmn_err(CE_CONT, "?TSC calibrated using %s; freq is %lu MHz\n", tsccpp[i]->tscc_source, tsc_freq / 1000000); /* * Note that tsccpp is just a (sorted) array of * pointers to the tsc_calibration_t's (from the * linker set). The actual tsc_calibration_t's aren't * kmem_alloc()ed (being part of the linker set), so * it's safe to keep a pointer to the one that was * used for calibration (intended for diagnostic * purposes). */ tsc_calibration_source = tsccpp[i]; kmem_free(tsccpp, tsc_set_size); tsc_pit_also(); return (tsc_freq); } } /* * In case it's useful, we don't free tsccpp -- we're about to panic * anyway. */ panic("Failed to calibrate TSC"); } uint64_t tsc_get_freq(void) { VERIFY(tsc_freq > 0); return (tsc_freq); }
