philipp/nixpkgs
1
0
Fork 0
Code Issues Pull requests Projects Releases Activity
nixpkgs/pkgs/os-specific/linux/kernel/patch-2.6.22-ck1

5168 lines
155 KiB
Plaintext
Raw Normal View History

I have added a special kernel 2.6.21 with the Con Kolivas patches that are designed to improve system responsiveness with specific emphasis on the desktop, but suitable to any workload. (and also the other patches like ext3cow and paravirt-nvidia) svn path=/nixpkgs/trunk/; revision=9748
2007-11-17 17:29:15 +01:00
Index: linux-2.6.22-ck1/include/linux/sched.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/sched.h 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/sched.h 2007-07-10 14:55:21.000000000 +1000
@@ -34,9 +34,14 @@
#define SCHED_FIFO 1
#define SCHED_RR 2
#define SCHED_BATCH 3
+#define SCHED_ISO 4
+#define SCHED_IDLEPRIO 5
#ifdef __KERNEL__
+#define SCHED_MAX SCHED_IDLEPRIO
+#define SCHED_RANGE(policy) ((policy) <= SCHED_MAX)
+
struct sched_param {
int sched_priority;
};
@@ -129,7 +134,7 @@
extern unsigned long nr_active(void);
extern unsigned long nr_iowait(void);
extern unsigned long weighted_cpuload(const int cpu);
-
+extern int above_background_load(void);
/*
* Task state bitmask. NOTE! These bits are also
@@ -150,8 +155,7 @@
#define EXIT_ZOMBIE 16
#define EXIT_DEAD 32
/* in tsk->state again */
-#define TASK_NONINTERACTIVE 64
-#define TASK_DEAD 128
+#define TASK_DEAD 64
#define __set_task_state(tsk, state_value) \
do { (tsk)->state = (state_value); } while (0)
@@ -537,14 +541,19 @@
#define MAX_USER_RT_PRIO 100
#define MAX_RT_PRIO MAX_USER_RT_PRIO
+#define PRIO_RANGE (40)
+#define ISO_PRIO (MAX_RT_PRIO - 1)
-#define MAX_PRIO (MAX_RT_PRIO + 40)
+#define MAX_PRIO (MAX_RT_PRIO + PRIO_RANGE)
-#define rt_prio(prio) unlikely((prio) < MAX_RT_PRIO)
+#define rt_prio(prio) unlikely((prio) < ISO_PRIO)
#define rt_task(p) rt_prio((p)->prio)
#define batch_task(p) (unlikely((p)->policy == SCHED_BATCH))
-#define is_rt_policy(p) ((p) != SCHED_NORMAL && (p) != SCHED_BATCH)
+#define is_rt_policy(policy) ((policy) == SCHED_FIFO || \
+ (policy) == SCHED_RR)
#define has_rt_policy(p) unlikely(is_rt_policy((p)->policy))
+#define iso_task(p) unlikely((p)->policy == SCHED_ISO)
+#define idleprio_task(p) unlikely((p)->policy == SCHED_IDLEPRIO)
/*
* Some day this will be a full-fledged user tracking system..
@@ -809,13 +818,6 @@
struct pipe_inode_info;
struct uts_namespace;
-enum sleep_type {
- SLEEP_NORMAL,
- SLEEP_NONINTERACTIVE,
- SLEEP_INTERACTIVE,
- SLEEP_INTERRUPTED,
-};
-
struct prio_array;
struct task_struct {
@@ -835,20 +837,33 @@
int load_weight; /* for niceness load balancing purposes */
int prio, static_prio, normal_prio;
struct list_head run_list;
+ /*
+ * This bitmap shows what priorities this task has received quota
+ * from for this major priority rotation on its current runqueue.
+ */
+ DECLARE_BITMAP(bitmap, PRIO_RANGE + 1);
struct prio_array *array;
+ /* Which major runqueue rotation did this task run */
+ unsigned long rotation;
unsigned short ioprio;
#ifdef CONFIG_BLK_DEV_IO_TRACE
unsigned int btrace_seq;
#endif
- unsigned long sleep_avg;
unsigned long long timestamp, last_ran;
unsigned long long sched_time; /* sched_clock time spent running */
- enum sleep_type sleep_type;
unsigned int policy;
cpumask_t cpus_allowed;
- unsigned int time_slice, first_time_slice;
+ /*
+ * How much this task is entitled to run at the current priority
+ * before being requeued at a lower priority.
+ */
+ int time_slice;
+ /* Is this the very first time_slice this task has ever run. */
+ unsigned int first_time_slice;
+ /* How much this task receives at each priority level */
+ int quota;
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
struct sched_info sched_info;
@@ -1013,6 +1028,7 @@
struct held_lock held_locks[MAX_LOCK_DEPTH];
unsigned int lockdep_recursion;
#endif
+ unsigned long mutexes_held;
/* journalling filesystem info */
void *journal_info;
@@ -1181,9 +1197,11 @@
#define PF_SWAPWRITE 0x00800000 /* Allowed to write to swap */
#define PF_SPREAD_PAGE 0x01000000 /* Spread page cache over cpuset */
#define PF_SPREAD_SLAB 0x02000000 /* Spread some slab caches over cpuset */
+#define PF_ISOREF 0x04000000 /* SCHED_ISO task has used up quota */
#define PF_MEMPOLICY 0x10000000 /* Non-default NUMA mempolicy */
#define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */
#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezeable */
+#define PF_NONSLEEP 0x80000000 /* Waiting on in-kernel activity */
/*
* Only the _current_ task can read/write to tsk->flags, but other
@@ -1253,7 +1271,7 @@
#endif
extern void set_user_nice(struct task_struct *p, long nice);
-extern int task_prio(const struct task_struct *p);
+extern int task_prio(struct task_struct *p);
extern int task_nice(const struct task_struct *p);
extern int can_nice(const struct task_struct *p, const int nice);
extern int task_curr(const struct task_struct *p);
Index: linux-2.6.22-ck1/kernel/sched.c
===================================================================
--- linux-2.6.22-ck1.orig/kernel/sched.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/kernel/sched.c 2007-07-10 14:55:24.000000000 +1000
@@ -16,6 +16,7 @@
* by Davide Libenzi, preemptible kernel bits by Robert Love.
* 2003-09-03 Interactivity tuning by Con Kolivas.
* 2004-04-02 Scheduler domains code by Nick Piggin
+ * 2007-03-02 Staircase deadline scheduling policy by Con Kolivas
*/
#include <linux/mm.h>
@@ -53,8 +54,9 @@
#include <linux/kprobes.h>
#include <linux/delayacct.h>
#include <linux/reciprocal_div.h>
-
+#include <linux/log2.h>
#include <asm/tlb.h>
+
#include <asm/unistd.h>
/*
@@ -84,147 +86,85 @@
#define USER_PRIO(p) ((p)-MAX_RT_PRIO)
#define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio)
#define MAX_USER_PRIO (USER_PRIO(MAX_PRIO))
+#define SCHED_PRIO(p) ((p)+MAX_RT_PRIO)
-/*
- * Some helpers for converting nanosecond timing to jiffy resolution
- */
-#define NS_TO_JIFFIES(TIME) ((TIME) / (1000000000 / HZ))
+/* Some helpers for converting to/from various scales.*/
#define JIFFIES_TO_NS(TIME) ((TIME) * (1000000000 / HZ))
-
-/*
- * These are the 'tuning knobs' of the scheduler:
- *
- * Minimum timeslice is 5 msecs (or 1 jiffy, whichever is larger),
- * default timeslice is 100 msecs, maximum timeslice is 800 msecs.
- * Timeslices get refilled after they expire.
- */
-#define MIN_TIMESLICE max(5 * HZ / 1000, 1)
-#define DEF_TIMESLICE (100 * HZ / 1000)
-#define ON_RUNQUEUE_WEIGHT 30
-#define CHILD_PENALTY 95
-#define PARENT_PENALTY 100
-#define EXIT_WEIGHT 3
-#define PRIO_BONUS_RATIO 25
-#define MAX_BONUS (MAX_USER_PRIO * PRIO_BONUS_RATIO / 100)
-#define INTERACTIVE_DELTA 2
-#define MAX_SLEEP_AVG (DEF_TIMESLICE * MAX_BONUS)
-#define STARVATION_LIMIT (MAX_SLEEP_AVG)
-#define NS_MAX_SLEEP_AVG (JIFFIES_TO_NS(MAX_SLEEP_AVG))
-
-/*
- * If a task is 'interactive' then we reinsert it in the active
- * array after it has expired its current timeslice. (it will not
- * continue to run immediately, it will still roundrobin with
- * other interactive tasks.)
- *
- * This part scales the interactivity limit depending on niceness.
- *
- * We scale it linearly, offset by the INTERACTIVE_DELTA delta.
- * Here are a few examples of different nice levels:
- *
- * TASK_INTERACTIVE(-20): [1,1,1,1,1,1,1,1,1,0,0]
- * TASK_INTERACTIVE(-10): [1,1,1,1,1,1,1,0,0,0,0]
- * TASK_INTERACTIVE( 0): [1,1,1,1,0,0,0,0,0,0,0]
- * TASK_INTERACTIVE( 10): [1,1,0,0,0,0,0,0,0,0,0]
- * TASK_INTERACTIVE( 19): [0,0,0,0,0,0,0,0,0,0,0]
- *
- * (the X axis represents the possible -5 ... 0 ... +5 dynamic
- * priority range a task can explore, a value of '1' means the
- * task is rated interactive.)
- *
- * Ie. nice +19 tasks can never get 'interactive' enough to be
- * reinserted into the active array. And only heavily CPU-hog nice -20
- * tasks will be expired. Default nice 0 tasks are somewhere between,
- * it takes some effort for them to get interactive, but it's not
- * too hard.
- */
-
-#define CURRENT_BONUS(p) \
- (NS_TO_JIFFIES((p)->sleep_avg) * MAX_BONUS / \
- MAX_SLEEP_AVG)
-
-#define GRANULARITY (10 * HZ / 1000 ? : 1)
-
-#ifdef CONFIG_SMP
-#define TIMESLICE_GRANULARITY(p) (GRANULARITY * \
- (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1)) * \
- num_online_cpus())
-#else
-#define TIMESLICE_GRANULARITY(p) (GRANULARITY * \
- (1 << (((MAX_BONUS - CURRENT_BONUS(p)) ? : 1) - 1)))
-#endif
-
-#define SCALE(v1,v1_max,v2_max) \
- (v1) * (v2_max) / (v1_max)
-
-#define DELTA(p) \
- (SCALE(TASK_NICE(p) + 20, 40, MAX_BONUS) - 20 * MAX_BONUS / 40 + \
- INTERACTIVE_DELTA)
-
-#define TASK_INTERACTIVE(p) \
- ((p)->prio <= (p)->static_prio - DELTA(p))
-
-#define INTERACTIVE_SLEEP(p) \
- (JIFFIES_TO_NS(MAX_SLEEP_AVG * \
- (MAX_BONUS / 2 + DELTA((p)) + 1) / MAX_BONUS - 1))
-
-#define TASK_PREEMPTS_CURR(p, rq) \
- ((p)->prio < (rq)->curr->prio)
-
-#define SCALE_PRIO(x, prio) \
- max(x * (MAX_PRIO - prio) / (MAX_USER_PRIO / 2), MIN_TIMESLICE)
-
-static unsigned int static_prio_timeslice(int static_prio)
-{
- if (static_prio < NICE_TO_PRIO(0))
- return SCALE_PRIO(DEF_TIMESLICE * 4, static_prio);
- else
- return SCALE_PRIO(DEF_TIMESLICE, static_prio);
-}
-
-#ifdef CONFIG_SMP
-/*
- * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
- * Since cpu_power is a 'constant', we can use a reciprocal divide.
+#define MS_TO_NS(TIME) ((TIME) * 1000000)
+#define MS_TO_US(TIME) ((TIME) * 1000)
+#define US_TO_MS(TIME) ((TIME) / 1000)
+
+#define TASK_PREEMPTS_CURR(p, curr) ((p)->prio < (curr)->prio)
+
+/*
+ * This is the time all tasks within the same priority round robin.
+ * Value is in ms and set to a minimum of 10ms. Scales with number of cpus.
+ * Tunable via /proc interface.
+ */
+int rr_interval __read_mostly = 6;
+int sched_interactive __read_mostly = 1;
+
+/*
+ * sched_iso_cpu - sysctl which determines the cpu percentage SCHED_ISO tasks
+ * are allowed to run (over ISO_PERIOD seconds) as real time tasks.
+ * sched_iso_period - sysctl which determines the number of seconds over
+ * which cpu usage of SCHED_ISO tasks is averaged to determine if they are
+ * exceeding their allowable bandwidth.
+*/
+int sched_iso_cpu __read_mostly = 80;
+int sched_iso_period __read_mostly = 5;
+
+#define ISO_PERIOD ((sched_iso_period * HZ) + 1)
+
+/*
+ * This contains a bitmap for each dynamic priority level with empty slots
+ * for the valid priorities each different nice level can have. It allows
+ * us to stagger the slots where differing priorities run in a way that
+ * keeps latency differences between different nice levels at a minimum.
+ * The purpose of a pre-generated matrix is for rapid lookup of next slot in
+ * O(1) time without having to recalculate every time priority gets demoted.
+ * All nice levels use priority slot 39 as this allows less niced tasks to
+ * get all priority slots better than that before expiration is forced.
+ * ie, where 0 means a slot for that priority, priority running from left to
+ * right is from prio 0 to prio 39:
+ * nice -20 0000000000000000000000000000000000000000
+ * nice -10 1000100010001000100010001000100010010000
+ * nice 0 1010101010101010101010101010101010101010
+ * nice 5 1011010110110101101101011011010110110110
+ * nice 10 1110111011101110111011101110111011101110
+ * nice 15 1111111011111110111111101111111011111110
+ * nice 19 1111111111111111111111111111111111111110
*/
-static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
-{
- return reciprocal_divide(load, sg->reciprocal_cpu_power);
-}
+static unsigned long prio_matrix[PRIO_RANGE][BITS_TO_LONGS(PRIO_RANGE)]
+ __read_mostly;
-/*
- * Each time a sched group cpu_power is changed,
- * we must compute its reciprocal value
- */
-static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
-{
- sg->__cpu_power += val;
- sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
-}
-#endif
+struct rq;
/*
- * task_timeslice() scales user-nice values [ -20 ... 0 ... 19 ]
- * to time slice values: [800ms ... 100ms ... 5ms]
- *
- * The higher a thread's priority, the bigger timeslices
- * it gets during one round of execution. But even the lowest
- * priority thread gets MIN_TIMESLICE worth of execution time.
+ * These are the runqueue data structures:
*/
+struct prio_array {
+ /* Tasks queued at each priority */
+ struct list_head queue[MAX_PRIO + 1];
-static inline unsigned int task_timeslice(struct task_struct *p)
-{
- return static_prio_timeslice(p->static_prio);
-}
+ /*
+ * The bitmap of priorities queued for this array. While the expired
+ * array will never have realtime tasks on it, it is simpler to have
+ * equal sized bitmaps for a cheap array swap. Include 1 bit for
+ * delimiter.
+ */
+ DECLARE_BITMAP(prio_bitmap, MAX_PRIO + 1);
-/*
- * These are the runqueue data structures:
- */
+ /*
+ * The best static priority (of the dynamic priority tasks) queued
+ * this array.
+ */
+ int best_static_prio;
-struct prio_array {
- unsigned int nr_active;
- DECLARE_BITMAP(bitmap, MAX_PRIO+1); /* include 1 bit for delimiter */
- struct list_head queue[MAX_PRIO];
+#ifdef CONFIG_SMP
+ /* For convenience looks back at rq */
+ struct rq *rq;
+#endif
};
/*
@@ -260,14 +200,28 @@
*/
unsigned long nr_uninterruptible;
- unsigned long expired_timestamp;
/* Cached timestamp set by update_cpu_clock() */
unsigned long long most_recent_timestamp;
struct task_struct *curr, *idle;
unsigned long next_balance;
struct mm_struct *prev_mm;
- struct prio_array *active, *expired, arrays[2];
- int best_expired_prio;
+
+ struct prio_array *active, *expired, *idleprio, arrays[2];
+ unsigned long *dyn_bitmap, *exp_bitmap;
+
+ /*
+ * The current dynamic priority level this runqueue is at per static
+ * priority level.
+ */
+ int prio_level[PRIO_RANGE];
+
+ /* How many times we have rotated the priority queue */
+ unsigned long prio_rotation;
+ unsigned long iso_ticks;
+ unsigned short iso_refractory;
+
+ /* Number of idleprio tasks running */
+ unsigned long nr_idleprio;
atomic_t nr_iowait;
#ifdef CONFIG_SMP
@@ -606,12 +560,9 @@
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
/*
* Called when a process is dequeued from the active array and given
- * the cpu. We should note that with the exception of interactive
- * tasks, the expired queue will become the active queue after the active
- * queue is empty, without explicitly dequeuing and requeuing tasks in the
- * expired queue. (Interactive tasks may be requeued directly to the
- * active queue, thus delaying tasks in the expired queue from running;
- * see scheduler_tick()).
+ * the cpu. We should note that the expired queue will become the active
+ * queue after the active queue is empty, without explicitly dequeuing and
+ * requeuing tasks in the expired queue.
*
* This function is only called from sched_info_arrive(), rather than
* dequeue_task(). Even though a task may be queued and dequeued multiple
@@ -709,71 +660,304 @@
#define sched_info_switch(t, next) do { } while (0)
#endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
+static int idleprio_suitable(struct task_struct *p)
+{
+ return (!p->mutexes_held && !freezing(p) && !signal_pending(p) &&
+ !(p->flags & (PF_NONSLEEP | PF_EXITING)));
+}
+
+static int idleprio(const struct task_struct *p)
+{
+ return (p->prio == MAX_PRIO);
+}
+
+static inline int task_queued(struct task_struct *task)
+{
+ return !list_empty(&task->run_list);
+}
+
+static inline void set_dynamic_bit(struct task_struct *p, struct rq *rq)
+{
+ __set_bit(p->prio, p->array->prio_bitmap);
+}
+
/*
- * Adding/removing a task to/from a priority array:
+ * Removing from a runqueue.
*/
-static void dequeue_task(struct task_struct *p, struct prio_array *array)
+static void dequeue_task(struct task_struct *p, struct rq *rq)
{
- array->nr_active--;
- list_del(&p->run_list);
- if (list_empty(array->queue + p->prio))
- __clear_bit(p->prio, array->bitmap);
+ list_del_init(&p->run_list);
+ if (idleprio_task(p) && idleprio(p))
+ rq->nr_idleprio--;
+ else if (list_empty(p->array->queue + p->prio))
+ __clear_bit(p->prio, p->array->prio_bitmap);
}
-static void enqueue_task(struct task_struct *p, struct prio_array *array)
+static void reset_first_time_slice(struct task_struct *p)
{
- sched_info_queued(p);
- list_add_tail(&p->run_list, array->queue + p->prio);
- __set_bit(p->prio, array->bitmap);
- array->nr_active++;
+ if (unlikely(p->first_time_slice))
+ p->first_time_slice = 0;
+}
+
+/*
+ * The task is being queued on a fresh array so it has its entitlement
+ * bitmap cleared.
+ */
+static void task_new_array(struct task_struct *p, struct rq *rq,
+ struct prio_array *array)
+{
+ bitmap_zero(p->bitmap, PRIO_RANGE);
+ p->rotation = rq->prio_rotation;
+ p->time_slice = p->quota;
p->array = array;
+ reset_first_time_slice(p);
+}
+
+/* Find the first slot from the relevant prio_matrix entry */
+static int first_prio_slot(struct task_struct *p)
+{
+ if (unlikely(p->policy == SCHED_BATCH))
+ return p->static_prio;
+ return SCHED_PRIO(find_first_zero_bit(
+ prio_matrix[USER_PRIO(p->static_prio)], PRIO_RANGE));
}
/*
- * Put task to the end of the run list without the overhead of dequeue
- * followed by enqueue.
+ * In sched_interactive mode priority allocation occurs per process per rq
+ * array swap. In !sched_interactive mode all waking tasks must obey the
+ * current prio level of all other tasks running per array swap.
*/
-static void requeue_task(struct task_struct *p, struct prio_array *array)
+static int minprio(struct rq *rq, int uprio)
{
- list_move_tail(&p->run_list, array->queue + p->prio);
+ if (sched_interactive)
+ return MAX_RT_PRIO;
+ return rq->prio_level[uprio];
}
-static inline void
-enqueue_task_head(struct task_struct *p, struct prio_array *array)
+/*
+ * Find the first unused slot by this task that is also in its prio_matrix
+ * level. SCHED_BATCH tasks do not use the priority matrix. They only take
+ * priority slots from their static_prio and above.
+ */
+static int next_entitled_slot(struct task_struct *p, struct rq *rq)
{
- list_add(&p->run_list, array->queue + p->prio);
- __set_bit(p->prio, array->bitmap);
- array->nr_active++;
- p->array = array;
+ int search_prio = MAX_RT_PRIO, uprio = USER_PRIO(p->static_prio);
+ struct prio_array *array = rq->active;
+ DECLARE_BITMAP(tmp, PRIO_RANGE);
+
+ /*
+ * Go straight to expiration if there are higher priority tasks
+ * already expired.
+ */
+ if (p->static_prio > rq->expired->best_static_prio)
+ return MAX_PRIO;
+ if (!rq->prio_level[uprio])
+ rq->prio_level[uprio] = MAX_RT_PRIO;
+ /*
+ * Only priorities equal to the prio_level and above for their
+ * static_prio are acceptable, and only if it's not better than
+ * a queued better static_prio's prio_level.
+ */
+ if (p->static_prio < array->best_static_prio) {
+ if (likely(p->policy != SCHED_BATCH))
+ array->best_static_prio = p->static_prio;
+ } else if (p->static_prio == array->best_static_prio) {
+ search_prio = minprio(rq, uprio);
+ } else {
+ int i;
+
+ search_prio = minprio(rq, uprio);
+ /* A bound O(n) function, worst case n is 40 */
+ for (i = array->best_static_prio; i <= p->static_prio ; i++) {
+ if (!rq->prio_level[USER_PRIO(i)])
+ rq->prio_level[USER_PRIO(i)] = MAX_RT_PRIO;
+ search_prio = max(search_prio,
+ rq->prio_level[USER_PRIO(i)]);
+ }
+ }
+ if (unlikely(p->policy == SCHED_BATCH)) {
+ search_prio = max(search_prio, p->static_prio);
+ return SCHED_PRIO(find_next_zero_bit(p->bitmap, PRIO_RANGE,
+ USER_PRIO(search_prio)));
+ }
+ bitmap_or(tmp, p->bitmap, prio_matrix[uprio], PRIO_RANGE);
+ return SCHED_PRIO(find_next_zero_bit(tmp, PRIO_RANGE,
+ USER_PRIO(search_prio)));
+}
+
+static void queue_expired(struct task_struct *p, struct rq *rq)
+{
+ task_new_array(p, rq, rq->expired);
+ p->prio = p->normal_prio = first_prio_slot(p);
+ if (p->static_prio < rq->expired->best_static_prio)
+ rq->expired->best_static_prio = p->static_prio;
+ reset_first_time_slice(p);
}
+#ifdef CONFIG_SMP
/*
- * __normal_prio - return the priority that is based on the static
- * priority but is modified by bonuses/penalties.
- *
- * We scale the actual sleep average [0 .... MAX_SLEEP_AVG]
- * into the -5 ... 0 ... +5 bonus/penalty range.
- *
- * We use 25% of the full 0...39 priority range so that:
- *
- * 1) nice +19 interactive tasks do not preempt nice 0 CPU hogs.
- * 2) nice -20 CPU hogs do not get preempted by nice 0 tasks.
- *
- * Both properties are important to certain workloads.
+ * If we're waking up a task that was previously on a different runqueue,
+ * update its data appropriately. Note we may be reading data from src_rq->
+ * outside of lock, but the occasional inaccurate result should be harmless.
*/
+ static void update_if_moved(struct task_struct *p, struct rq *rq)
+{
+ struct rq *src_rq = p->array->rq;
+
+ if (src_rq == rq)
+ return;
+ /*
+ * Only need to set p->array when p->rotation == rq->prio_rotation as
+ * they will be set in recalc_task_prio when != rq->prio_rotation.
+ */
+ if (p->rotation == src_rq->prio_rotation) {
+ p->rotation = rq->prio_rotation;
+ if (p->array == src_rq->expired)
+ p->array = rq->expired;
+ else
+ p->array = rq->active;
+ } else
+ p->rotation = 0;
+}
+#else
+static inline void update_if_moved(struct task_struct *p, struct rq *rq)
+{
+}
+#endif
-static inline int __normal_prio(struct task_struct *p)
+static inline int isoprio_suitable(struct task_struct *p)
{
- int bonus, prio;
+ return !(p->flags & PF_ISOREF);
+}
- bonus = CURRENT_BONUS(p) - MAX_BONUS / 2;
+static int task_timeslice(struct task_struct *p);
- prio = p->static_prio - bonus;
- if (prio < MAX_RT_PRIO)
- prio = MAX_RT_PRIO;
- if (prio > MAX_PRIO-1)
- prio = MAX_PRIO-1;
- return prio;
+/*
+ * recalc_task_prio determines what priority a non rt_task will be
+ * queued at. If the task has already been running during this runqueue's
+ * major rotation (rq->prio_rotation) then it continues at the same
+ * priority if it has tick entitlement left. If it does not have entitlement
+ * left, it finds the next priority slot according to its nice value that it
+ * has not extracted quota from. If it has not run during this major
+ * rotation, it starts at the next_entitled_slot and has its bitmap quota
+ * cleared. If it does not have any slots left it has all its slots reset and
+ * is queued on the expired at its first_prio_slot.
+ */
+static void recalc_task_prio(struct task_struct *p, struct rq *rq)
+{
+ struct prio_array *array = rq->active;
+ int queue_prio;
+
+ if (iso_task(p)) {
+ if (isoprio_suitable(p)) {
+ /*
+ * If SCHED_ISO tasks have not used up their real time
+ * quota they have run just better than highest
+ * SCHED_NORMAL priority. Otherwise they run as
+ * SCHED_NORMAL.
+ */
+ p->prio = p->normal_prio = ISO_PRIO;
+ p->array = rq->active;
+ if (p->time_slice <= 0)
+ p->time_slice = p->quota;
+ return;
+ } else if (p->prio == ISO_PRIO) {
+ /* Just about to be demoted to SCHED_NORMAL */
+ p->time_slice = 0;
+ }
+ } else if (idleprio_task(p)) {
+ if (idleprio_suitable(p)) {
+ /*
+ * If suitable idleprio_tasks are queued at MAX_PRIO
+ * only on the idleprio array. Their time_slice is
+ * their full task_timeslice as they cooperatively
+ * multitask.
+ */
+ p->prio = p->normal_prio = MAX_PRIO;
+ p->array = rq->idleprio;
+ if (p->time_slice <= 0)
+ p->time_slice = task_timeslice(p);
+ return;
+ }
+ /*
+ * If unsuitable idleprio_tasks are queued equivalent to
+ * nice 19 tasks on the expired array.
+ */
+ p->flags &= ~PF_NONSLEEP;
+ p->prio = p->normal_prio = MAX_PRIO - 1;
+ p->array = rq->expired;
+ if (p->time_slice <= 0 || p->time_slice > p->quota)
+ p->time_slice = p->quota;
+ return;
+ }
+
+ update_if_moved(p, rq);
+ if (p->rotation == rq->prio_rotation) {
+ if (p->array == array) {
+ if (p->time_slice > 0)
+ return;
+ p->time_slice = p->quota;
+ } else if (p->array == rq->expired) {
+ queue_expired(p, rq);
+ return;
+ } else
+ task_new_array(p, rq, array);
+ } else
+ task_new_array(p, rq, array);
+
+ queue_prio = next_entitled_slot(p, rq);
+ if (queue_prio >= MAX_PRIO) {
+ queue_expired(p, rq);
+ return;
+ }
+ p->prio = p->normal_prio = queue_prio;
+ __set_bit(USER_PRIO(p->prio), p->bitmap);
+}
+
+/*
+ * Adding to a runqueue. The dynamic priority queue that it is added to is
+ * determined by recalc_task_prio() above.
+ */
+static inline void __enqueue_task(struct task_struct *p, struct rq *rq)
+{
+ if (rt_task(p))
+ p->array = rq->active;
+ else
+ recalc_task_prio(p, rq);
+
+ if (idleprio_task(p) && idleprio(p))
+ rq->nr_idleprio++;
+ sched_info_queued(p);
+ set_dynamic_bit(p, rq);
+}
+
+static void enqueue_task(struct task_struct *p, struct rq *rq)
+{
+ __enqueue_task(p, rq);
+ list_add_tail(&p->run_list, p->array->queue + p->prio);
+}
+
+static inline void enqueue_task_head(struct task_struct *p, struct rq *rq)
+{
+ __enqueue_task(p, rq);
+ list_add(&p->run_list, p->array->queue + p->prio);
+}
+
+/*
+ * requeue_task is only called when p->static_prio does not change. p->prio
+ * can change with dynamic tasks.
+ */
+static void requeue_task(struct task_struct *p, struct rq *rq,
+ struct prio_array *old_array, int old_prio)
+{
+ if (p->array == rq->expired)
+ queue_expired(p, rq);
+ list_move_tail(&p->run_list, p->array->queue + p->prio);
+ if (!rt_task(p)) {
+ if (list_empty(old_array->queue + old_prio))
+ __clear_bit(old_prio, old_array->prio_bitmap);
+ set_dynamic_bit(p, rq);
+ }
}
/*
@@ -786,20 +970,29 @@
*/
/*
- * Assume: static_prio_timeslice(NICE_TO_PRIO(0)) == DEF_TIMESLICE
- * If static_prio_timeslice() is ever changed to break this assumption then
- * this code will need modification
- */
-#define TIME_SLICE_NICE_ZERO DEF_TIMESLICE
-#define LOAD_WEIGHT(lp) \
- (((lp) * SCHED_LOAD_SCALE) / TIME_SLICE_NICE_ZERO)
-#define PRIO_TO_LOAD_WEIGHT(prio) \
- LOAD_WEIGHT(static_prio_timeslice(prio))
-#define RTPRIO_TO_LOAD_WEIGHT(rp) \
- (PRIO_TO_LOAD_WEIGHT(MAX_RT_PRIO) + LOAD_WEIGHT(rp))
+ * task_timeslice - the total duration a task can run during one major
+ * rotation. Returns value in milliseconds as the smallest value can be 1.
+ */
+static int task_timeslice(struct task_struct *p)
+{
+ int slice = p->quota; /* quota is in us */
+
+ if (!rt_task(p))
+ slice += (PRIO_RANGE - 1 - TASK_USER_PRIO(p)) * slice;
+ return US_TO_MS(slice);
+}
+
+/*
+ * The load weight is basically the task_timeslice in ms. Realtime tasks are
+ * special cased to be proportionately larger than nice -20 by their
+ * rt_priority. The weight for rt tasks can only be arbitrary at best.
+ */
+#define RTPRIO_TO_LOAD_WEIGHT(rp) (rr_interval * 20 * (40 + rp))
static void set_load_weight(struct task_struct *p)
{
+ int load_weight;
+
if (has_rt_policy(p)) {
#ifdef CONFIG_SMP
if (p == task_rq(p)->migration_thread)
@@ -808,12 +1001,19 @@
* Giving its load any weight will skew balancing
* adversely.
*/
- p->load_weight = 0;
+ load_weight = 0;
else
#endif
- p->load_weight = RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);
+ load_weight = RTPRIO_TO_LOAD_WEIGHT(p->rt_priority);
} else
- p->load_weight = PRIO_TO_LOAD_WEIGHT(p->static_prio);
+ load_weight = task_timeslice(p);
+ /*
+ * idleprio tasks have much lower weight than SCHED_NORMAL tasks but
+ * still need to be weighted to allow balancing to occur.
+ */
+ if (likely(!idleprio_task(p)))
+ load_weight *= PRIO_RANGE;
+ p->load_weight = load_weight;
}
static inline void
@@ -841,28 +1041,38 @@
}
/*
- * Calculate the expected normal priority: i.e. priority
- * without taking RT-inheritance into account. Might be
- * boosted by interactivity modifiers. Changes upon fork,
- * setprio syscalls, and whenever the interactivity
- * estimator recalculates.
+ * __activate_task - move a task to the runqueue.
*/
-static inline int normal_prio(struct task_struct *p)
+static inline void __activate_task(struct task_struct *p, struct rq *rq)
{
- int prio;
+ enqueue_task(p, rq);
+ inc_nr_running(p, rq);
+}
+/*
+ * __activate_idle_task - move idle task to the _front_ of runqueue.
+ */
+static inline void __activate_idle_task(struct task_struct *p, struct rq *rq)
+{
+ enqueue_task_head(p, rq);
+ inc_nr_running(p, rq);
+}
+
+static inline int normal_prio(struct task_struct *p)
+{
if (has_rt_policy(p))
- prio = MAX_RT_PRIO-1 - p->rt_priority;
+ return MAX_RT_PRIO-1 - p->rt_priority;
+ /* Other tasks all have normal_prio set in recalc_task_prio */
+ if (likely(p->prio >= MAX_RT_PRIO && p->prio < MAX_PRIO))
+ return p->prio;
else
- prio = __normal_prio(p);
- return prio;
+ return p->static_prio;
}
/*
* Calculate the current priority, i.e. the priority
* taken into account by the scheduler. This value might
- * be boosted by RT tasks, or might be boosted by
- * interactivity modifiers. Will be RT if the task got
+ * be boosted by RT tasks as it will be RT if the task got
* RT-boosted. If not then it returns p->normal_prio.
*/
static int effective_prio(struct task_struct *p)
@@ -878,112 +1088,70 @@
return p->prio;
}
-/*
- * __activate_task - move a task to the runqueue.
- */
-static void __activate_task(struct task_struct *p, struct rq *rq)
+static inline unsigned int nice_quota_ms(int nice)
{
- struct prio_array *target = rq->active;
+ unsigned int rr = rr_interval;
- if (batch_task(p))
- target = rq->expired;
- enqueue_task(p, target);
- inc_nr_running(p, rq);
+ if (nice < -6) {
+ rr *= nice * nice;
+ rr /= 40;
+ } else if (nice > 0)
+ rr = rr / 2 ? : 1;
+ return rr;
}
+#define DEFAULT_WEIGHT (nice_quota_ms(0) * 20 * PRIO_RANGE)
+
/*
- * __activate_idle_task - move idle task to the _front_ of runqueue.
+ * A runqueue laden with a single nice 0 task scores a weighted_cpuload of
+ * SCHED_LOAD_SCALE. This function returns 1 if any cpu is laden with a
+ * task of nice 0 or enough lower priority tasks to bring up the
+ * weighted_cpuload
*/
-static inline void __activate_idle_task(struct task_struct *p, struct rq *rq)
+int above_background_load(void)
{
- enqueue_task_head(p, rq->active);
- inc_nr_running(p, rq);
+ unsigned long cpu;
+
+ for_each_online_cpu(cpu) {
+ if (weighted_cpuload(cpu) >= DEFAULT_WEIGHT)
+ return 1;
+ }
+ return 0;
}
/*
- * Recalculate p->normal_prio and p->prio after having slept,
- * updating the sleep-average too:
+ * All tasks have quotas based on rr_interval. RT tasks all get rr_interval.
+ * From nice 1 to 19 they are smaller than it only if they are at least one
+ * tick still. Below nice 0 they get progressively larger.
+ * ie nice -6..0 = rr_interval. nice -10 = 2.5 * rr_interval
+ * nice -20 = 10 * rr_interval. nice 1-19 = rr_interval / 2.
+ * Value returned is in microseconds.
*/
-static int recalc_task_prio(struct task_struct *p, unsigned long long now)
+static inline unsigned int rr_quota(struct task_struct *p)
{
- /* Caller must always ensure 'now >= p->timestamp' */
- unsigned long sleep_time = now - p->timestamp;
+ unsigned int quota;
- if (batch_task(p))
- sleep_time = 0;
-
- if (likely(sleep_time > 0)) {
- /*
- * This ceiling is set to the lowest priority that would allow
- * a task to be reinserted into the active array on timeslice
- * completion.
- */
- unsigned long ceiling = INTERACTIVE_SLEEP(p);
-
- if (p->mm && sleep_time > ceiling && p->sleep_avg < ceiling) {
- /*
- * Prevents user tasks from achieving best priority
- * with one single large enough sleep.
- */
- p->sleep_avg = ceiling;
- /*
- * Using INTERACTIVE_SLEEP() as a ceiling places a
- * nice(0) task 1ms sleep away from promotion, and
- * gives it 700ms to round-robin with no chance of
- * being demoted. This is more than generous, so
- * mark this sleep as non-interactive to prevent the
- * on-runqueue bonus logic from intervening should
- * this task not receive cpu immediately.
- */
- p->sleep_type = SLEEP_NONINTERACTIVE;
- } else {
- /*
- * Tasks waking from uninterruptible sleep are
- * limited in their sleep_avg rise as they
- * are likely to be waiting on I/O
- */
- if (p->sleep_type == SLEEP_NONINTERACTIVE && p->mm) {
- if (p->sleep_avg >= ceiling)
- sleep_time = 0;
- else if (p->sleep_avg + sleep_time >=
- ceiling) {
- p->sleep_avg = ceiling;
- sleep_time = 0;
- }
- }
-
- /*
- * This code gives a bonus to interactive tasks.
- *
- * The boost works by updating the 'average sleep time'
- * value here, based on ->timestamp. The more time a
- * task spends sleeping, the higher the average gets -
- * and the higher the priority boost gets as well.
- */
- p->sleep_avg += sleep_time;
-
- }
- if (p->sleep_avg > NS_MAX_SLEEP_AVG)
- p->sleep_avg = NS_MAX_SLEEP_AVG;
- }
+ if (rt_task(p))
+ quota = rr_interval;
+ else
+ quota = nice_quota_ms(TASK_NICE(p));
+ return MS_TO_US(quota);
+}
- return effective_prio(p);
+/* Every time we set the quota we need to set the load weight */
+static void set_quota(struct task_struct *p)
+{
+ p->quota = rr_quota(p);
+ set_load_weight(p);
}
/*
* activate_task - move a task to the runqueue and do priority recalculation
- *
- * Update all the scheduling statistics stuff. (sleep average
- * calculation, priority modifiers, etc.)
*/
static void activate_task(struct task_struct *p, struct rq *rq, int local)
{
- unsigned long long now;
-
- if (rt_task(p))
- goto out;
+ unsigned long long now = sched_clock();
- now = sched_clock();
#ifdef CONFIG_SMP
if (!local) {
/* Compensate for drifting sched_clock */
@@ -1004,32 +1172,9 @@
(now - p->timestamp) >> 20);
}
- p->prio = recalc_task_prio(p, now);
-
- /*
- * This checks to make sure it's not an uninterruptible task
- * that is now waking up.
- */
- if (p->sleep_type == SLEEP_NORMAL) {
- /*
- * Tasks which were woken up by interrupts (ie. hw events)
- * are most likely of interactive nature. So we give them
- * the credit of extending their sleep time to the period
- * of time they spend on the runqueue, waiting for execution
- * on a CPU, first time around:
- */
- if (in_interrupt())
- p->sleep_type = SLEEP_INTERRUPTED;
- else {
- /*
- * Normal first-time wakeups get a credit too for
- * on-runqueue time, but it will be weighted down:
- */
- p->sleep_type = SLEEP_INTERACTIVE;
- }
- }
+ set_quota(p);
+ p->prio = effective_prio(p);
p->timestamp = now;
-out:
__activate_task(p, rq);
}
@@ -1039,8 +1184,7 @@
static void deactivate_task(struct task_struct *p, struct rq *rq)
{
dec_nr_running(p, rq);
- dequeue_task(p, p->array);
- p->array = NULL;
+ dequeue_task(p, rq);
}
/*
@@ -1133,7 +1277,7 @@
* If the task is not on a runqueue (and not running), then
* it is sufficient to simply update the task's cpu field.
*/
- if (!p->array && !task_running(rq, p)) {
+ if (!task_queued(p) && !task_running(rq, p)) {
set_task_cpu(p, dest_cpu);
return 0;
}
@@ -1159,7 +1303,6 @@
{
unsigned long flags;
struct rq *rq;
- struct prio_array *array;
int running;
repeat:
@@ -1192,7 +1335,6 @@
*/
rq = task_rq_lock(p, &flags);
running = task_running(rq, p);
- array = p->array;
task_rq_unlock(rq, &flags);
/*
@@ -1215,7 +1357,7 @@
* running right now), it's preempted, and we should
* yield - it could be a while.
*/
- if (unlikely(array)) {
+ if (unlikely(task_queued(p))) {
yield();
goto repeat;
}
@@ -1294,6 +1436,25 @@
}
/*
+ * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
+ * Since cpu_power is a 'constant', we can use a reciprocal divide.
+ */
+static inline u32 sg_div_cpu_power(const struct sched_group *sg, u32 load)
+{
+ return reciprocal_divide(load, sg->reciprocal_cpu_power);
+}
+
+/*
+ * Each time a sched group cpu_power is changed,
+ * we must compute its reciprocal value
+ */
+static inline void sg_inc_cpu_power(struct sched_group *sg, u32 val)
+{
+ sg->__cpu_power += val;
+ sg->reciprocal_cpu_power = reciprocal_value(sg->__cpu_power);
+}
+
+/*
* find_idlest_group finds and returns the least busy CPU group within the
* domain.
*/
@@ -1490,6 +1651,31 @@
}
#endif
+/*
+ * We need to have a special definition for an idle runqueue when testing
+ * for preemption on CONFIG_HOTPLUG_CPU as the idle task may be scheduled as
+ * a realtime task in sched_idle_next.
+ */
+#ifdef CONFIG_HOTPLUG_CPU
+#define rq_idle(rq) ((rq)->curr == (rq)->idle && !rt_task((rq)->curr))
+#else
+#define rq_idle(rq) ((rq)->curr == (rq)->idle)
+#endif
+
+static inline int task_preempts_curr(struct task_struct *p, struct rq *rq)
+{
+ struct task_struct *curr = rq->curr;
+
+ return ((p->array == task_rq(p)->active &&
+ TASK_PREEMPTS_CURR(p, curr)) || rq_idle(rq));
+}
+
+static inline void try_preempt(struct task_struct *p, struct rq *rq)
+{
+ if (task_preempts_curr(p, rq))
+ resched_task(rq->curr);
+}
+
/***
* try_to_wake_up - wake up a thread
* @p: the to-be-woken-up thread
@@ -1521,7 +1707,7 @@
if (!(old_state & state))
goto out;
- if (p->array)
+ if (task_queued(p))
goto out_running;
cpu = task_cpu(p);
@@ -1614,7 +1800,7 @@
old_state = p->state;
if (!(old_state & state))
goto out;
- if (p->array)
+ if (task_queued(p))
goto out_running;
this_cpu = smp_processor_id();
@@ -1623,25 +1809,9 @@
out_activate:
#endif /* CONFIG_SMP */
- if (old_state == TASK_UNINTERRUPTIBLE) {
+ if (old_state == TASK_UNINTERRUPTIBLE)
rq->nr_uninterruptible--;
- /*
- * Tasks on involuntary sleep don't earn
- * sleep_avg beyond just interactive state.
- */
- p->sleep_type = SLEEP_NONINTERACTIVE;
- } else
-
- /*
- * Tasks that have marked their sleep as noninteractive get
- * woken up with their sleep average not weighted in an
- * interactive way.
- */
- if (old_state & TASK_NONINTERACTIVE)
- p->sleep_type = SLEEP_NONINTERACTIVE;
-
- activate_task(p, rq, cpu == this_cpu);
/*
* Sync wakeups (i.e. those types of wakeups where the waker
* has indicated that it will leave the CPU in short order)
@@ -1650,15 +1820,22 @@
* the waker guarantees that the freshly woken up task is going
* to be considered on this CPU.)
*/
- if (!sync || cpu != this_cpu) {
- if (TASK_PREEMPTS_CURR(p, rq))
- resched_task(rq->curr);
- }
+ activate_task(p, rq, cpu == this_cpu);
+ if (!sync || cpu != this_cpu)
+ try_preempt(p, rq);
success = 1;
out_running:
p->state = TASK_RUNNING;
out:
+ /*
+ * Special case when freezing we need to reschedule idleprio tasks
+ * as SCHED_NORMAL or else they'll never freeze
+ */
+ if (idleprio_task(p) && freezing(p) && idleprio(p)) {
+ dequeue_task(p, rq);
+ enqueue_task(p, rq);
+ }
task_rq_unlock(rq, &flags);
return success;
@@ -1676,7 +1853,6 @@
return try_to_wake_up(p, state, 0);
}
-static void task_running_tick(struct rq *rq, struct task_struct *p);
/*
* Perform scheduler related setup for a newly forked process p.
* p is forked by current.
@@ -1704,7 +1880,6 @@
p->prio = current->normal_prio;
INIT_LIST_HEAD(&p->run_list);
- p->array = NULL;
#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
if (unlikely(sched_info_on()))
memset(&p->sched_info, 0, sizeof(p->sched_info));
@@ -1716,30 +1891,31 @@
/* Want to start with kernel preemption disabled. */
task_thread_info(p)->preempt_count = 1;
#endif
+ if (unlikely(p->policy == SCHED_FIFO))
+ goto out;
/*
* Share the timeslice between parent and child, thus the
* total amount of pending timeslices in the system doesn't change,
* resulting in more scheduling fairness.
*/
local_irq_disable();
- p->time_slice = (current->time_slice + 1) >> 1;
- /*
- * The remainder of the first timeslice might be recovered by
- * the parent if the child exits early enough.
- */
- p->first_time_slice = 1;
- current->time_slice >>= 1;
- p->timestamp = sched_clock();
- if (unlikely(!current->time_slice)) {
+ if (current->time_slice > 0) {
+ current->time_slice /= 2;
+ if (current->time_slice)
+ p->time_slice = current->time_slice;
+ else
+ p->time_slice = 1;
/*
- * This case is rare, it happens when the parent has only
- * a single jiffy left from its timeslice. Taking the
- * runqueue lock is not a problem.
+ * The remainder of the first timeslice might be recovered by
+ * the parent if the child exits early enough.
*/
- current->time_slice = 1;
- task_running_tick(cpu_rq(cpu), current);
- }
+ p->first_time_slice = 1;
+ } else
+ p->time_slice = 0;
+
+ p->timestamp = sched_clock();
local_irq_enable();
+out:
put_cpu();
}
@@ -1761,38 +1937,16 @@
this_cpu = smp_processor_id();
cpu = task_cpu(p);
- /*
- * We decrease the sleep average of forking parents
- * and children as well, to keep max-interactive tasks
- * from forking tasks that are max-interactive. The parent
- * (current) is done further down, under its lock.
- */
- p->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(p) *
- CHILD_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS);
-
- p->prio = effective_prio(p);
-
if (likely(cpu == this_cpu)) {
+ activate_task(p, rq, 1);
if (!(clone_flags & CLONE_VM)) {
/*
* The VM isn't cloned, so we're in a good position to
* do child-runs-first in anticipation of an exec. This
* usually avoids a lot of COW overhead.
*/
- if (unlikely(!current->array))
- __activate_task(p, rq);
- else {
- p->prio = current->prio;
- p->normal_prio = current->normal_prio;
- list_add_tail(&p->run_list, &current->run_list);
- p->array = current->array;
- p->array->nr_active++;
- inc_nr_running(p, rq);
- }
set_need_resched();
- } else
- /* Run child last */
- __activate_task(p, rq);
+ }
/*
* We skip the following code due to cpu == this_cpu
*
@@ -1809,19 +1963,16 @@
*/
p->timestamp = (p->timestamp - this_rq->most_recent_timestamp)
+ rq->most_recent_timestamp;
- __activate_task(p, rq);
- if (TASK_PREEMPTS_CURR(p, rq))
- resched_task(rq->curr);
+ activate_task(p, rq, 0);
+ try_preempt(p, rq);
/*
* Parent and child are on different CPUs, now get the
- * parent runqueue to update the parent's ->sleep_avg:
+ * parent runqueue to update the parent's ->flags:
*/
task_rq_unlock(rq, &flags);
this_rq = task_rq_lock(current, &flags);
}
- current->sleep_avg = JIFFIES_TO_NS(CURRENT_BONUS(current) *
- PARENT_PENALTY / 100 * MAX_SLEEP_AVG / MAX_BONUS);
task_rq_unlock(this_rq, &flags);
}
@@ -1836,23 +1987,17 @@
*/
void fastcall sched_exit(struct task_struct *p)
{
+ struct task_struct *parent;
unsigned long flags;
struct rq *rq;
- /*
- * If the child was a (relative-) CPU hog then decrease
- * the sleep_avg of the parent as well.
- */
- rq = task_rq_lock(p->parent, &flags);
- if (p->first_time_slice && task_cpu(p) == task_cpu(p->parent)) {
- p->parent->time_slice += p->time_slice;
- if (unlikely(p->parent->time_slice > task_timeslice(p)))
- p->parent->time_slice = task_timeslice(p);
- }
- if (p->sleep_avg < p->parent->sleep_avg)
- p->parent->sleep_avg = p->parent->sleep_avg /
- (EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->sleep_avg /
- (EXIT_WEIGHT + 1);
+ parent = p->parent;
+ rq = task_rq_lock(parent, &flags);
+ if (p->first_time_slice > 0 && task_cpu(p) == task_cpu(parent)) {
+ parent->time_slice += p->time_slice;
+ if (unlikely(parent->time_slice > parent->quota))
+ parent->time_slice = parent->quota;
+ }
task_rq_unlock(rq, &flags);
}
@@ -2184,23 +2329,17 @@
* pull_task - move a task from a remote runqueue to the local runqueue.
* Both runqueues must be locked.
*/
-static void pull_task(struct rq *src_rq, struct prio_array *src_array,
- struct task_struct *p, struct rq *this_rq,
- struct prio_array *this_array, int this_cpu)
+static void pull_task(struct rq *src_rq, struct task_struct *p,
+ struct rq *this_rq, int this_cpu)
{
- dequeue_task(p, src_array);
+ dequeue_task(p, src_rq);
dec_nr_running(p, src_rq);
set_task_cpu(p, this_cpu);
inc_nr_running(p, this_rq);
- enqueue_task(p, this_array);
+ enqueue_task(p, this_rq);
p->timestamp = (p->timestamp - src_rq->most_recent_timestamp)
+ this_rq->most_recent_timestamp;
- /*
- * Note that idle threads have a prio of MAX_PRIO, for this test
- * to be always true for them.
- */
- if (TASK_PREEMPTS_CURR(p, this_rq))
- resched_task(this_rq->curr);
+ try_preempt(p, this_rq);
}
/*
@@ -2243,7 +2382,16 @@
return 1;
}
-#define rq_best_prio(rq) min((rq)->curr->prio, (rq)->best_expired_prio)
+static inline int rq_best_prio(struct rq *rq)
+{
+ int best_prio, exp_prio;
+
+ best_prio = sched_find_first_bit(rq->dyn_bitmap);
+ exp_prio = find_next_bit(rq->exp_bitmap, MAX_PRIO, MAX_RT_PRIO);
+ if (unlikely(best_prio > exp_prio))
+ best_prio = exp_prio;
+ return best_prio;
+}
/*
* move_tasks tries to move up to max_nr_move tasks and max_load_move weighted
@@ -2259,7 +2407,7 @@
{
int idx, pulled = 0, pinned = 0, this_best_prio, best_prio,
best_prio_seen, skip_for_load;
- struct prio_array *array, *dst_array;
+ struct prio_array *array;
struct list_head *head, *curr;
struct task_struct *tmp;
long rem_load_move;
@@ -2286,31 +2434,29 @@
* be cache-cold, thus switching CPUs has the least effect
* on them.
*/
- if (busiest->expired->nr_active) {
- array = busiest->expired;
- dst_array = this_rq->expired;
- } else {
- array = busiest->active;
- dst_array = this_rq->active;
- }
-
+ array = busiest->expired;
new_array:
- /* Start searching at priority 0: */
- idx = 0;
+ /* Expired arrays don't have RT tasks so they're always MAX_RT_PRIO+ */
+ if (array == busiest->expired)
+ idx = MAX_RT_PRIO;
+ else
+ idx = 0;
skip_bitmap:
if (!idx)
- idx = sched_find_first_bit(array->bitmap);
+ idx = sched_find_first_bit(array->prio_bitmap);
else
- idx = find_next_bit(array->bitmap, MAX_PRIO, idx);
- if (idx >= MAX_PRIO) {
- if (array == busiest->expired && busiest->active->nr_active) {
+ idx = find_next_bit(array->prio_bitmap, MAX_PRIO, idx);
+ if (idx == MAX_PRIO) {
+ if (array == busiest->idleprio && busiest->nr_idleprio)
+ goto found_idleprio;
+ if (array == busiest->expired) {
array = busiest->active;
- dst_array = this_rq->active;
goto new_array;
}
goto out;
}
+found_idleprio:
head = array->queue + idx;
curr = head->prev;
skip_queue:
@@ -2332,11 +2478,22 @@
best_prio_seen |= idx == best_prio;
if (curr != head)
goto skip_queue;
+ if (idx == MAX_PRIO) {
+ /*
+ * Occurs either when balancing idleprio tasks or
+ * there really are no more tasks to find.
+ */
+ if (array == busiest->expired) {
+ array = busiest->active;
+ goto new_array;
+ }
+ goto out;
+ }
idx++;
goto skip_bitmap;
}
- pull_task(busiest, array, tmp, this_rq, dst_array, this_cpu);
+ pull_task(busiest, tmp, this_rq, this_cpu);
pulled++;
rem_load_move -= tmp->load_weight;
@@ -2349,6 +2506,13 @@
this_best_prio = idx;
if (curr != head)
goto skip_queue;
+ if (idx == MAX_PRIO) {
+ if (array == busiest->expired) {
+ array = busiest->active;
+ goto new_array;
+ }
+ goto out;
+ }
idx++;
goto skip_bitmap;
}
@@ -3297,11 +3461,36 @@
/*
* This is called on clock ticks and on context switches.
* Bank in p->sched_time the ns elapsed since the last tick or switch.
+ * CPU scheduler quota accounting is also performed here in microseconds.
+ * The value returned from sched_clock() occasionally gives bogus values so
+ * some sanity checking is required.
*/
-static inline void
-update_cpu_clock(struct task_struct *p, struct rq *rq, unsigned long long now)
+static void
+update_cpu_clock(struct task_struct *p, struct rq *rq, unsigned long long now,
+ int tick)
{
- p->sched_time += now - p->last_ran;
+ long time_diff = now - p->last_ran;
+
+ if (tick) {
+ /*
+ * Called from scheduler_tick() there should be less than two
+ * jiffies worth, and not negative/overflow.
+ */
+ if (time_diff > JIFFIES_TO_NS(2) || time_diff < 0)
+ time_diff = JIFFIES_TO_NS(1);
+ } else {
+ /*
+ * Called from context_switch there should be less than one
+ * jiffy worth, and not negative/overflow. There should be
+ * some time banked here so use a nominal 1us.
+ */
+ if (time_diff > JIFFIES_TO_NS(1) || time_diff < 1)
+ time_diff = 1000;
+ }
+ /* time_slice accounting is done in usecs to avoid overflow on 32bit */
+ if (p != rq->idle && p->policy != SCHED_FIFO)
+ p->time_slice -= time_diff / 1000;
+ p->sched_time += time_diff;
p->last_ran = rq->most_recent_timestamp = now;
}
@@ -3322,27 +3511,6 @@
}
/*
- * We place interactive tasks back into the active array, if possible.
- *
- * To guarantee that this does not starve expired tasks we ignore the
- * interactivity of a task if the first expired task had to wait more
- * than a 'reasonable' amount of time. This deadline timeout is
- * load-dependent, as the frequency of array switched decreases with
- * increasing number of running tasks. We also ignore the interactivity
- * if a better static_prio task has expired:
- */
-static inline int expired_starving(struct rq *rq)
-{
- if (rq->curr->static_prio > rq->best_expired_prio)
- return 1;
- if (!STARVATION_LIMIT || !rq->expired_timestamp)
- return 0;
- if (jiffies - rq->expired_timestamp > STARVATION_LIMIT * rq->nr_running)
- return 1;
- return 0;
-}
-
-/*
* Account user cpu time to a process.
* @p: the process that the cpu time gets accounted to
* @hardirq_offset: the offset to subtract from hardirq_count()
@@ -3357,7 +3525,7 @@
/* Add user time to cpustat. */
tmp = cputime_to_cputime64(cputime);
- if (TASK_NICE(p) > 0)
+ if (TASK_NICE(p) > 0 || idleprio_task(p))
cpustat->nice = cputime64_add(cpustat->nice, tmp);
else
cpustat->user = cputime64_add(cpustat->user, tmp);
@@ -3415,87 +3583,94 @@
cpustat->steal = cputime64_add(cpustat->steal, tmp);
}
-static void task_running_tick(struct rq *rq, struct task_struct *p)
+/*
+ * The task has used up its quota of running in this prio_level so it must be
+ * dropped a priority level, all managed by recalc_task_prio().
+ */
+static void task_expired_entitlement(struct rq *rq, struct task_struct *p)
{
- if (p->array != rq->active) {
- /* Task has expired but was not scheduled yet */
- set_tsk_need_resched(p);
+ int overrun;
+
+ reset_first_time_slice(p);
+ if (rt_task(p)) {
+ p->time_slice += p->quota;
+ list_move_tail(&p->run_list, p->array->queue + p->prio);
return;
}
- spin_lock(&rq->lock);
+ overrun = p->time_slice;
+ dequeue_task(p, rq);
+ enqueue_task(p, rq);
/*
- * The task was running during this tick - update the
- * time slice counter. Note: we do not update a thread's
- * priority until it either goes to sleep or uses up its
- * timeslice. This makes it possible for interactive tasks
- * to use up their timeslices at their highest priority levels.
+ * Subtract any extra time this task ran over its time_slice; ie
+ * overrun will either be 0 or negative.
*/
- if (rt_task(p)) {
- /*
- * RR tasks need a special form of timeslice management.
- * FIFO tasks have no timeslices.
- */
- if ((p->policy == SCHED_RR) && !--p->time_slice) {
- p->time_slice = task_timeslice(p);
- p->first_time_slice = 0;
- set_tsk_need_resched(p);
+ p->time_slice += overrun;
+}
- /* put it at the end of the queue: */
- requeue_task(p, rq->active);
- }
- goto out_unlock;
+/*
+ * Test if SCHED_ISO tasks have run longer than their alloted period as RT
+ * tasks and set the refractory flag if necessary. There is 10% hysteresis
+ * for unsetting the flag.
+ */
+static unsigned int test_ret_isorefractory(struct rq *rq)
+{
+ if (likely(!rq->iso_refractory)) {
+ if (rq->iso_ticks / ISO_PERIOD > sched_iso_cpu)
+ rq->iso_refractory = 1;
+ } else {
+ if (rq->iso_ticks / ISO_PERIOD < (sched_iso_cpu * 90 / 100))
+ rq->iso_refractory = 0;
}
- if (!--p->time_slice) {
- dequeue_task(p, rq->active);
- set_tsk_need_resched(p);
- p->prio = effective_prio(p);
- p->time_slice = task_timeslice(p);
- p->first_time_slice = 0;
+ return rq->iso_refractory;
+}
- if (!rq->expired_timestamp)
- rq->expired_timestamp = jiffies;
- if (!TASK_INTERACTIVE(p) || expired_starving(rq)) {
- enqueue_task(p, rq->expired);
- if (p->static_prio < rq->best_expired_prio)
- rq->best_expired_prio = p->static_prio;
- } else
- enqueue_task(p, rq->active);
- } else {
- /*
- * Prevent a too long timeslice allowing a task to monopolize
- * the CPU. We do this by splitting up the timeslice into
- * smaller pieces.
- *
- * Note: this does not mean the task's timeslices expire or
- * get lost in any way, they just might be preempted by
- * another task of equal priority. (one with higher
- * priority would have preempted this task already.) We
- * requeue this task to the end of the list on this priority
- * level, which is in essence a round-robin of tasks with
- * equal priority.
- *
- * This only applies to tasks in the interactive
- * delta range with at least TIMESLICE_GRANULARITY to requeue.
- */
- if (TASK_INTERACTIVE(p) && !((task_timeslice(p) -
- p->time_slice) % TIMESLICE_GRANULARITY(p)) &&
- (p->time_slice >= TIMESLICE_GRANULARITY(p)) &&
- (p->array == rq->active)) {
+/* No SCHED_ISO task was running so decrease rq->iso_ticks */
+static inline void no_iso_tick(struct rq *rq)
+{
+ rq->iso_ticks = rq->iso_ticks * (ISO_PERIOD - 1) / ISO_PERIOD;
+}
- requeue_task(p, rq->active);
- set_tsk_need_resched(p);
- }
+/* This manages tasks that have run out of timeslice during a scheduler_tick */
+static void task_running_tick(struct rq *rq, struct task_struct *p)
+{
+ /*
+ * If a SCHED_ISO task is running we increment the iso_ticks. In
+ * order to prevent SCHED_ISO tasks from causing starvation in the
+ * presence of true RT tasks we account those as iso_ticks as well.
+ */
+ if ((rt_task(p) || (iso_task(p) && !rq->iso_refractory))) {
+ if (rq->iso_ticks <= (ISO_PERIOD * 100) - 100)
+ rq->iso_ticks += 100;
+ } else
+ no_iso_tick(rq);
+
+ if (iso_task(p)) {
+ if (unlikely(test_ret_isorefractory(rq))) {
+ if (isoprio_suitable(p)) {
+ /*
+ * SCHED_ISO task is running as RT and limit
+ * has been hit. Set the PF_ISOREF flag and
+ * force it to reschedule as SCHED_NORMAL
+ * by zeroing its time_slice
+ */
+ p->flags |= PF_ISOREF;
+ p->time_slice = 0;
+ }
+ } else
+ p->flags &= ~PF_ISOREF;
}
-out_unlock:
- spin_unlock(&rq->lock);
+ /* SCHED_FIFO tasks never run out of timeslice. */
+ if (p->time_slice > 0 || p->policy == SCHED_FIFO)
+ return;
+ /* p->time_slice <= 0 */
+ set_tsk_need_resched(p);
+ if (likely(task_queued(p)))
+ task_expired_entitlement(rq, p);
}
/*
* This function gets called by the timer code, with HZ frequency.
* We call it with interrupts disabled.
- *
- * It also gets called by the fork code, when changing the parent's
- * timeslices.
*/
void scheduler_tick(void)
{
@@ -3505,10 +3680,14 @@
int idle_at_tick = idle_cpu(cpu);
struct rq *rq = cpu_rq(cpu);
- update_cpu_clock(p, rq, now);
+ update_cpu_clock(p, rq, now, 1);
+ spin_lock(&rq->lock);
if (!idle_at_tick)
task_running_tick(rq, p);
+ else
+ no_iso_tick(rq);
+ spin_unlock(&rq->lock);
#ifdef CONFIG_SMP
update_load(rq);
rq->idle_at_tick = idle_at_tick;
@@ -3554,10 +3733,80 @@
#endif
-static inline int interactive_sleep(enum sleep_type sleep_type)
+static void reset_prio_levels(struct rq *rq)
{
- return (sleep_type == SLEEP_INTERACTIVE ||
- sleep_type == SLEEP_INTERRUPTED);
+ rq->active->best_static_prio = MAX_PRIO - 1;
+ rq->expired->best_static_prio = MAX_PRIO - 1;
+ memset(rq->prio_level, 0, sizeof(int) * PRIO_RANGE);
+}
+
+/*
+ * Only tasks running are SCHED_IDLEPRIO. Set the active array to the
+ * idleprio array and if it isn't already active
+ */
+static struct task_struct *next_idleprio_task(struct rq *rq)
+{
+ struct prio_array *array = rq->active;
+ struct list_head *queue;
+
+ if (array != rq->idleprio) {
+ rq->active = rq->idleprio;
+ rq->expired = array;
+ array = rq->active;
+ rq->exp_bitmap = rq->expired->prio_bitmap;
+ rq->dyn_bitmap = rq->active->prio_bitmap;
+ }
+ rq->prio_rotation++;
+ reset_prio_levels(rq);
+ queue = array->queue + MAX_PRIO;
+ return list_entry(queue->next, struct task_struct, run_list);
+}
+
+/*
+ * next_dynamic_task finds the next suitable dynamic task.
+ */
+static inline struct task_struct *next_dynamic_task(struct rq *rq, int idx)
+{
+ struct prio_array *array = rq->active;
+ struct task_struct *next;
+ struct list_head *queue;
+ int nstatic;
+
+retry:
+ if (unlikely(rq->nr_running == rq->nr_idleprio))
+ return next_idleprio_task(rq);
+ if (idx >= MAX_PRIO) {
+ /* There are no more tasks in the active array. Swap arrays */
+ array = rq->expired;
+ rq->expired = rq->active;
+ rq->active = array;
+ rq->exp_bitmap = rq->expired->prio_bitmap;
+ rq->dyn_bitmap = rq->active->prio_bitmap;
+ rq->prio_rotation++;
+ idx = find_next_bit(rq->dyn_bitmap, MAX_PRIO, MAX_RT_PRIO);
+ reset_prio_levels(rq);
+ }
+ queue = array->queue + idx;
+ next = list_entry(queue->next, struct task_struct, run_list);
+ if (unlikely(next->time_slice <= 0 && !(iso_task(next) &&
+ isoprio_suitable(next)))) {
+ /*
+ * Unlucky enough that this task ran out of time_slice
+ * before it hit a scheduler_tick so it should have its
+ * priority reassessed and choose another task (possibly
+ * the same one)
+ */
+ task_expired_entitlement(rq, next);
+ idx = find_next_bit(rq->dyn_bitmap, MAX_PRIO, MAX_RT_PRIO);
+ goto retry;
+ }
+ next->rotation = rq->prio_rotation;
+ nstatic = next->static_prio;
+ if (nstatic < array->best_static_prio)
+ array->best_static_prio = nstatic;
+ if (idx > rq->prio_level[USER_PRIO(nstatic)])
+ rq->prio_level[USER_PRIO(nstatic)] = idx;
+ return next;
}
/*
@@ -3566,13 +3815,11 @@
asmlinkage void __sched schedule(void)
{
struct task_struct *prev, *next;
- struct prio_array *array;
struct list_head *queue;
unsigned long long now;
- unsigned long run_time;
- int cpu, idx, new_prio;
long *switch_count;
struct rq *rq;
+ int cpu, idx;
/*
* Test if we are atomic. Since do_exit() needs to call into
@@ -3608,18 +3855,6 @@
schedstat_inc(rq, sched_cnt);
now = sched_clock();
- if (likely((long long)(now - prev->timestamp) < NS_MAX_SLEEP_AVG)) {
- run_time = now - prev->timestamp;
- if (unlikely((long long)(now - prev->timestamp) < 0))
- run_time = 0;
- } else
- run_time = NS_MAX_SLEEP_AVG;
-
- /*
- * Tasks charged proportionately less run_time at high sleep_avg to
- * delay them losing their interactive status
- */
- run_time /= (CURRENT_BONUS(prev) ? : 1);
spin_lock_irq(&rq->lock);
@@ -3630,8 +3865,10 @@
unlikely(signal_pending(prev))))
prev->state = TASK_RUNNING;
else {
- if (prev->state == TASK_UNINTERRUPTIBLE)
+ if (prev->state == TASK_UNINTERRUPTIBLE) {
+ prev->flags |= PF_NONSLEEP;
rq->nr_uninterruptible++;
+ }
deactivate_task(prev, rq);
}
}
@@ -3641,59 +3878,29 @@
idle_balance(cpu, rq);
if (!rq->nr_running) {
next = rq->idle;
- rq->expired_timestamp = 0;
goto switch_tasks;
}
}
- array = rq->active;
- if (unlikely(!array->nr_active)) {
- /*
- * Switch the active and expired arrays.
- */
- schedstat_inc(rq, sched_switch);
- rq->active = rq->expired;
- rq->expired = array;
- array = rq->active;
- rq->expired_timestamp = 0;
- rq->best_expired_prio = MAX_PRIO;
- }
-
- idx = sched_find_first_bit(array->bitmap);
- queue = array->queue + idx;
- next = list_entry(queue->next, struct task_struct, run_list);
-
- if (!rt_task(next) && interactive_sleep(next->sleep_type)) {
- unsigned long long delta = now - next->timestamp;
- if (unlikely((long long)(now - next->timestamp) < 0))
- delta = 0;
-
- if (next->sleep_type == SLEEP_INTERACTIVE)
- delta = delta * (ON_RUNQUEUE_WEIGHT * 128 / 100) / 128;
-
- array = next->array;
- new_prio = recalc_task_prio(next, next->timestamp + delta);
-
- if (unlikely(next->prio != new_prio)) {
- dequeue_task(next, array);
- next->prio = new_prio;
- enqueue_task(next, array);
- }
+ idx = sched_find_first_bit(rq->dyn_bitmap);
+ if (likely(idx > ISO_PRIO))
+ next = next_dynamic_task(rq, idx);
+ else {
+ queue = rq->active->queue + idx;
+ next = list_entry(queue->next, struct task_struct, run_list);
}
- next->sleep_type = SLEEP_NORMAL;
switch_tasks:
- if (next == rq->idle)
+ if (next == rq->idle) {
+ reset_prio_levels(rq);
+ rq->prio_rotation++;
schedstat_inc(rq, sched_goidle);
+ }
prefetch(next);
prefetch_stack(next);
clear_tsk_need_resched(prev);
rcu_qsctr_inc(task_cpu(prev));
- update_cpu_clock(prev, rq, now);
-
- prev->sleep_avg -= run_time;
- if ((long)prev->sleep_avg <= 0)
- prev->sleep_avg = 0;
+ update_cpu_clock(prev, rq, now, 0);
prev->timestamp = prev->last_ran = now;
sched_info_switch(prev, next);
@@ -4129,29 +4336,22 @@
*/
void rt_mutex_setprio(struct task_struct *p, int prio)
{
- struct prio_array *array;
unsigned long flags;
+ int queued, oldprio;
struct rq *rq;
- int oldprio;
BUG_ON(prio < 0 || prio > MAX_PRIO);
rq = task_rq_lock(p, &flags);
oldprio = p->prio;
- array = p->array;
- if (array)
- dequeue_task(p, array);
+ queued = task_queued(p);
+ if (queued)
+ dequeue_task(p, rq);
p->prio = prio;
- if (array) {
- /*
- * If changing to an RT priority then queue it
- * in the active array!
- */
- if (rt_task(p))
- array = rq->active;
- enqueue_task(p, array);
+ if (queued) {
+ enqueue_task(p, rq);
/*
* Reschedule if we are currently running on this runqueue and
* our priority decreased, or if we are not currently running on
@@ -4160,8 +4360,8 @@
if (task_running(rq, p)) {
if (p->prio > oldprio)
resched_task(rq->curr);
- } else if (TASK_PREEMPTS_CURR(p, rq))
- resched_task(rq->curr);
+ } else
+ try_preempt(p, rq);
}
task_rq_unlock(rq, &flags);
}
@@ -4170,8 +4370,7 @@
void set_user_nice(struct task_struct *p, long nice)
{
- struct prio_array *array;
- int old_prio, delta;
+ int queued, old_prio,delta;
unsigned long flags;
struct rq *rq;
@@ -4192,26 +4391,27 @@
p->static_prio = NICE_TO_PRIO(nice);
goto out_unlock;
}
- array = p->array;
- if (array) {
- dequeue_task(p, array);
+ queued = task_queued(p);
+ if (queued) {
+ dequeue_task(p, rq);
dec_raw_weighted_load(rq, p);
}
p->static_prio = NICE_TO_PRIO(nice);
- set_load_weight(p);
old_prio = p->prio;
p->prio = effective_prio(p);
+ set_quota(p);
delta = p->prio - old_prio;
- if (array) {
- enqueue_task(p, array);
+ if (queued) {
+ enqueue_task(p, rq);
inc_raw_weighted_load(rq, p);
/*
* If the task increased its priority or is running and
* lowered its priority, then reschedule its CPU:
*/
- if (delta < 0 || (delta > 0 && task_running(rq, p)))
+ if (delta < 0 || ((delta > 0 || idleprio_task(p)) &&
+ task_running(rq, p)))
resched_task(rq->curr);
}
out_unlock:
@@ -4281,11 +4481,23 @@
*
* This is the priority value as seen by users in /proc.
* RT tasks are offset by -200. Normal tasks are centered
- * around 0, value goes from -16 to +15.
+ * around 1, value goes from 0 to +79. Values higher than
+ * 39 indicate task is on the expired array. This is done
+ * lockless and may rarely return an active instead of
+ * expired value.
*/
-int task_prio(const struct task_struct *p)
+int task_prio(struct task_struct *p)
{
- return p->prio - MAX_RT_PRIO;
+ int prio = p->prio - MAX_RT_PRIO;
+
+ if (task_queued(p)) {
+ struct rq *rq = task_rq(p);
+ struct prio_array *array = p->array;
+
+ if (rq && rq->expired == array)
+ prio += PRIO_RANGE;
+ }
+ return prio;
}
/**
@@ -4328,19 +4540,14 @@
/* Actually do priority change: must hold rq lock. */
static void __setscheduler(struct task_struct *p, int policy, int prio)
{
- BUG_ON(p->array);
+ BUG_ON(task_queued(p));
p->policy = policy;
p->rt_priority = prio;
p->normal_prio = normal_prio(p);
/* we are holding p->pi_lock already */
p->prio = rt_mutex_getprio(p);
- /*
- * SCHED_BATCH tasks are treated as perpetual CPU hogs:
- */
- if (policy == SCHED_BATCH)
- p->sleep_avg = 0;
- set_load_weight(p);
+ set_quota(p);
}
/**
@@ -4354,19 +4561,36 @@
int sched_setscheduler(struct task_struct *p, int policy,
struct sched_param *param)
{
- int retval, oldprio, oldpolicy = -1;
- struct prio_array *array;
+ struct sched_param zero_param = { .sched_priority = 0 };
+ int queued, retval, oldprio, oldpolicy = -1;
+ unsigned long rlim_rtprio = 0;
unsigned long flags;
struct rq *rq;
/* may grab non-irq protected spin_locks */
BUG_ON(in_interrupt());
+ if (is_rt_policy(policy) && !capable(CAP_SYS_NICE)) {
+ unsigned long lflags;
+
+ if (!lock_task_sighand(p, &lflags))
+ return -ESRCH;
+ rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
+ unlock_task_sighand(p, &lflags);
+ if (rlim_rtprio)
+ goto recheck;
+ /*
+ * If the caller requested an RT policy without having the
+ * necessary rights, we downgrade the policy to SCHED_ISO.
+ * We also set the parameter to zero to pass the checks.
+ */
+ policy = SCHED_ISO;
+ param = &zero_param;
+ }
recheck:
/* double check policy once rq lock held */
if (policy < 0)
policy = oldpolicy = p->policy;
- else if (policy != SCHED_FIFO && policy != SCHED_RR &&
- policy != SCHED_NORMAL && policy != SCHED_BATCH)
+ else if (!SCHED_RANGE(policy))
return -EINVAL;
/*
* Valid priorities for SCHED_FIFO and SCHED_RR are
@@ -4385,14 +4609,6 @@
*/
if (!capable(CAP_SYS_NICE)) {
if (is_rt_policy(policy)) {
- unsigned long rlim_rtprio;
- unsigned long flags;
-
- if (!lock_task_sighand(p, &flags))
- return -ESRCH;
- rlim_rtprio = p->signal->rlim[RLIMIT_RTPRIO].rlim_cur;
- unlock_task_sighand(p, &flags);
-
/* can't set/change the rt policy */
if (policy != p->policy && !rlim_rtprio)
return -EPERM;
@@ -4401,6 +4617,31 @@
if (param->sched_priority > p->rt_priority &&
param->sched_priority > rlim_rtprio)
return -EPERM;
+ } else {
+ switch (p->policy) {
+ /*
+ * Can only downgrade policies but not back to
+ * SCHED_NORMAL
+ */
+ case SCHED_ISO:
+ if (policy == SCHED_ISO)
+ goto out;
+ if (policy == SCHED_NORMAL)
+ return -EPERM;
+ break;
+ case SCHED_BATCH:
+ if (policy == SCHED_BATCH)
+ goto out;
+ if (policy != SCHED_IDLEPRIO)
+ return -EPERM;
+ break;
+ case SCHED_IDLEPRIO:
+ if (policy == SCHED_IDLEPRIO)
+ goto out;
+ return -EPERM;
+ default:
+ break;
+ }
}
/* can't change other user's priorities */
@@ -4409,6 +4650,11 @@
return -EPERM;
}
+ if (!(p->mm) && policy == SCHED_IDLEPRIO) {
+ /* Don't allow kernel threads to be SCHED_IDLEPRIO. */
+ return -EINVAL;
+ }
+
retval = security_task_setscheduler(p, policy, param);
if (retval)
return retval;
@@ -4429,12 +4675,12 @@
spin_unlock_irqrestore(&p->pi_lock, flags);
goto recheck;
}
- array = p->array;
- if (array)
+ queued = task_queued(p);
+ if (queued)
deactivate_task(p, rq);
oldprio = p->prio;
__setscheduler(p, policy, param->sched_priority);
- if (array) {
+ if (queued) {
__activate_task(p, rq);
/*
* Reschedule if we are currently running on this runqueue and
@@ -4444,14 +4690,15 @@
if (task_running(rq, p)) {
if (p->prio > oldprio)
resched_task(rq->curr);
- } else if (TASK_PREEMPTS_CURR(p, rq))
- resched_task(rq->curr);
+ } else
+ try_preempt(p, rq);
}
__task_rq_unlock(rq);
spin_unlock_irqrestore(&p->pi_lock, flags);
rt_mutex_adjust_pi(p);
+out:
return 0;
}
EXPORT_SYMBOL_GPL(sched_setscheduler);
@@ -4718,41 +4965,34 @@
* sys_sched_yield - yield the current processor to other threads.
*
* This function yields the current CPU by moving the calling thread
- * to the expired array. If there are no other threads running on this
- * CPU then this function will return.
+ * to the expired array if SCHED_NORMAL or the end of its current priority
+ * queue if a realtime task. If there are no other threads running on this
+ * cpu this function will return.
*/
asmlinkage long sys_sched_yield(void)
{
struct rq *rq = this_rq_lock();
- struct prio_array *array = current->array, *target = rq->expired;
+ struct task_struct *p = current;
schedstat_inc(rq, yld_cnt);
- /*
- * We implement yielding by moving the task into the expired
- * queue.
- *
- * (special rule: RT tasks will just roundrobin in the active
- * array.)
- */
- if (rt_task(current))
- target = rq->active;
-
- if (array->nr_active == 1) {
- schedstat_inc(rq, yld_act_empty);
- if (!rq->expired->nr_active)
- schedstat_inc(rq, yld_both_empty);
- } else if (!rq->expired->nr_active)
- schedstat_inc(rq, yld_exp_empty);
-
- if (array != target) {
- dequeue_task(current, array);
- enqueue_task(current, target);
- } else
- /*
- * requeue_task is cheaper so perform that if possible.
- */
- requeue_task(current, array);
+ if (rq->nr_running == 1)
+ schedstat_inc(rq, yld_both_empty);
+ else {
+ struct prio_array *old_array = p->array;
+ int old_prio = p->prio;
+
+ if (idleprio_task(p)) {
+ dequeue_task(p, rq);
+ enqueue_task(p, rq);
+ goto out_release;
+ }
+ /* p->prio will be updated in requeue_task via queue_expired */
+ if (!rt_task(p))
+ p->array = rq->expired;
+ requeue_task(p, rq, old_array, old_prio);
+ }
+out_release:
/*
* Since we are going to call schedule() anyway, there's
* no need to preempt or enable interrupts:
@@ -4902,6 +5142,8 @@
break;
case SCHED_NORMAL:
case SCHED_BATCH:
+ case SCHED_ISO:
+ case SCHED_IDLEPRIO:
ret = 0;
break;
}
@@ -4926,6 +5168,8 @@
break;
case SCHED_NORMAL:
case SCHED_BATCH:
+ case SCHED_ISO:
+ case SCHED_IDLEPRIO:
ret = 0;
}
return ret;
@@ -4959,8 +5203,8 @@
if (retval)
goto out_unlock;
- jiffies_to_timespec(p->policy == SCHED_FIFO ?
- 0 : task_timeslice(p), &t);
+ t = ns_to_timespec(p->policy == SCHED_FIFO ? 0 :
+ MS_TO_NS(task_timeslice(p)));
read_unlock(&tasklist_lock);
retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
out_nounlock:
@@ -5056,10 +5300,10 @@
struct rq *rq = cpu_rq(cpu);
unsigned long flags;
- idle->timestamp = sched_clock();
- idle->sleep_avg = 0;
- idle->array = NULL;
- idle->prio = idle->normal_prio = MAX_PRIO;
+ bitmap_zero(idle->bitmap, PRIO_RANGE);
+ idle->timestamp = idle->last_ran = sched_clock();
+ idle->array = rq->active;
+ idle->prio = idle->normal_prio = NICE_TO_PRIO(0);
idle->state = TASK_RUNNING;
idle->cpus_allowed = cpumask_of_cpu(cpu);
set_task_cpu(idle, cpu);
@@ -5178,7 +5422,7 @@
goto out;
set_task_cpu(p, dest_cpu);
- if (p->array) {
+ if (task_queued(p)) {
/*
* Sync timestamp with rq_dest's before activating.
* The same thing could be achieved by doing this step
@@ -5189,8 +5433,7 @@
+ rq_dest->most_recent_timestamp;
deactivate_task(p, rq_src);
__activate_task(p, rq_dest);
- if (TASK_PREEMPTS_CURR(p, rq_dest))
- resched_task(rq_dest->curr);
+ try_preempt(p, rq_dest);
}
ret = 1;
out:
@@ -5487,7 +5730,7 @@
/* Idle task back to normal (off runqueue, low prio) */
rq = task_rq_lock(rq->idle, &flags);
deactivate_task(rq->idle, rq);
- rq->idle->static_prio = MAX_PRIO;
+ rq->idle->static_prio = NICE_TO_PRIO(0);
__setscheduler(rq->idle, SCHED_NORMAL, 0);
migrate_dead_tasks(cpu);
task_rq_unlock(rq, &flags);
@@ -7013,6 +7256,13 @@
/* Move init over to a non-isolated CPU */
if (set_cpus_allowed(current, non_isolated_cpus) < 0)
BUG();
+
+ /*
+ * Assume that every added cpu gives us slightly less overall latency
+ * allowing us to increase the base rr_interval, but in a non linear
+ * fashion.
+ */
+ rr_interval *= 1 + ilog2(num_online_cpus());
}
#else
void __init sched_init_smp(void)
@@ -7035,6 +7285,16 @@
int i, j, k;
int highest_cpu = 0;
+ /* Generate the priority matrix */
+ for (i = 0; i < PRIO_RANGE; i++) {
+ bitmap_fill(prio_matrix[i], PRIO_RANGE);
+ j = PRIO_RANGE * PRIO_RANGE / (PRIO_RANGE - i);
+ for (k = 0; k <= PRIO_RANGE * (PRIO_RANGE - 1); k += j) {
+ __clear_bit(PRIO_RANGE - 1 - (k / PRIO_RANGE),
+ prio_matrix[i]);
+ }
+ }
+
for_each_possible_cpu(i) {
struct prio_array *array;
struct rq *rq;
@@ -7042,12 +7302,20 @@
rq = cpu_rq(i);
spin_lock_init(&rq->lock);
lockdep_set_class(&rq->lock, &rq->rq_lock_key);
+ rq->iso_ticks = 0;
rq->nr_running = 0;
+ rq->nr_idleprio = 0;
+ rq->prio_rotation = 0;
rq->active = rq->arrays;
+ rq->idleprio = rq->active;
rq->expired = rq->arrays + 1;
- rq->best_expired_prio = MAX_PRIO;
+ reset_prio_levels(rq);
+ rq->dyn_bitmap = rq->active->prio_bitmap;
+ rq->exp_bitmap = rq->expired->prio_bitmap;
#ifdef CONFIG_SMP
+ rq->active->rq = rq;
+ rq->expired->rq = rq;
rq->sd = NULL;
for (j = 1; j < 3; j++)
rq->cpu_load[j] = 0;
@@ -7060,17 +7328,16 @@
atomic_set(&rq->nr_iowait, 0);
for (j = 0; j < 2; j++) {
+
array = rq->arrays + j;
- for (k = 0; k < MAX_PRIO; k++) {
+ for (k = 0; k <= MAX_PRIO; k++)
INIT_LIST_HEAD(array->queue + k);
- __clear_bit(k, array->bitmap);
- }
- // delimiter for bitsearch
- __set_bit(MAX_PRIO, array->bitmap);
+ bitmap_zero(array->prio_bitmap, MAX_PRIO);
+ /* delimiter for bitsearch */
+ __set_bit(MAX_PRIO, array->prio_bitmap);
}
highest_cpu = i;
}
-
set_load_weight(&init_task);
#ifdef CONFIG_SMP
@@ -7125,25 +7392,25 @@
#ifdef CONFIG_MAGIC_SYSRQ
void normalize_rt_tasks(void)
{
- struct prio_array *array;
struct task_struct *g, *p;
unsigned long flags;
struct rq *rq;
+ int queued;
read_lock_irq(&tasklist_lock);
do_each_thread(g, p) {
- if (!rt_task(p))
+ if (!rt_task(p) && !iso_task(p))
continue;
spin_lock_irqsave(&p->pi_lock, flags);
rq = __task_rq_lock(p);
- array = p->array;
- if (array)
+ queued = task_queued(p);
+ if (queued)
deactivate_task(p, task_rq(p));
__setscheduler(p, SCHED_NORMAL, 0);
- if (array) {
+ if (queued) {
__activate_task(p, task_rq(p));
resched_task(rq->curr);
}
Index: linux-2.6.22-ck1/kernel/sysctl.c
===================================================================
--- linux-2.6.22-ck1.orig/kernel/sysctl.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/kernel/sysctl.c 2007-07-10 14:55:23.000000000 +1000
@@ -22,6 +22,7 @@
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/slab.h>
+#include <linux/swap-prefetch.h>
#include <linux/sysctl.h>
#include <linux/proc_fs.h>
#include <linux/capability.h>
@@ -70,6 +71,7 @@
extern char core_pattern[];
extern int pid_max;
extern int min_free_kbytes;
+extern int vm_tail_largefiles;
extern int printk_ratelimit_jiffies;
extern int printk_ratelimit_burst;
extern int pid_max_min, pid_max_max;
@@ -78,6 +80,10 @@
extern int compat_log;
extern int maps_protect;
extern int sysctl_stat_interval;
+extern int rr_interval;
+extern int sched_interactive;
+extern int sched_iso_cpu;
+extern int sched_iso_period;
/* this is needed for the proc_dointvec_minmax for [fs_]overflow UID and GID */
static int maxolduid = 65535;
@@ -161,6 +167,14 @@
#endif
+/* Constants for minimum and maximum testing.
+ We use these as one-element integer vectors. */
+static int __read_mostly zero;
+static int __read_mostly one = 1;
+static int __read_mostly one_hundred = 100;
+static int __read_mostly five_thousand = 5000;
+
+
/* The default sysctl tables: */
static ctl_table root_table[] = {
@@ -501,6 +515,47 @@
.mode = 0444,
.proc_handler = &proc_dointvec,
},
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "rr_interval",
+ .data = &rr_interval,
+ .maxlen = sizeof (int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec_minmax,
+ .strategy = &sysctl_intvec,
+ .extra1 = &one,
+ .extra2 = &five_thousand,
+ },
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "interactive",
+ .data = &sched_interactive,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "iso_cpu",
+ .data = &sched_iso_cpu,
+ .maxlen = sizeof (int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec_minmax,
+ .strategy = &sysctl_intvec,
+ .extra1 = &zero,
+ .extra2 = &one_hundred,
+ },
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "iso_period",
+ .data = &sched_iso_period,
+ .maxlen = sizeof (int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec_minmax,
+ .strategy = &sysctl_intvec,
+ .extra1 = &one,
+ .extra2 = &one_hundred,
+ },
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86)
{
.ctl_name = KERN_UNKNOWN_NMI_PANIC,
@@ -619,14 +674,16 @@
{ .ctl_name = 0 }
};
-/* Constants for minimum and maximum testing in vm_table.
- We use these as one-element integer vectors. */
-static int zero;
-static int one_hundred = 100;
-
-
static ctl_table vm_table[] = {
{
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "tail_largefiles",
+ .data = &vm_tail_largefiles,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
+ {
.ctl_name = VM_OVERCOMMIT_MEMORY,
.procname = "overcommit_memory",
.data = &sysctl_overcommit_memory,
@@ -705,16 +762,24 @@
.proc_handler = &proc_dointvec,
},
{
- .ctl_name = VM_SWAPPINESS,
- .procname = "swappiness",
- .data = &vm_swappiness,
- .maxlen = sizeof(vm_swappiness),
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "mapped",
+ .data = &vm_mapped,
+ .maxlen = sizeof(vm_mapped),
.mode = 0644,
.proc_handler = &proc_dointvec_minmax,
.strategy = &sysctl_intvec,
.extra1 = &zero,
.extra2 = &one_hundred,
},
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "hardmaplimit",
+ .data = &vm_hardmaplimit,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
#ifdef CONFIG_HUGETLB_PAGE
{
.ctl_name = VM_HUGETLB_PAGES,
@@ -882,6 +947,32 @@
.extra1 = &zero,
},
#endif
+#ifdef CONFIG_SWAP_PREFETCH
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "swap_prefetch",
+ .data = &swap_prefetch,
+ .maxlen = sizeof(swap_prefetch),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "swap_prefetch_delay",
+ .data = &swap_prefetch_delay,
+ .maxlen = sizeof(swap_prefetch_delay),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
+ {
+ .ctl_name = CTL_UNNUMBERED,
+ .procname = "swap_prefetch_sleep",
+ .data = &swap_prefetch_sleep,
+ .maxlen = sizeof(swap_prefetch_sleep),
+ .mode = 0644,
+ .proc_handler = &proc_dointvec,
+ },
+#endif
{ .ctl_name = 0 }
};
Index: linux-2.6.22-ck1/Documentation/sched-design.txt
===================================================================
--- linux-2.6.22-ck1.orig/Documentation/sched-design.txt 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/Documentation/sched-design.txt 2007-07-10 14:55:02.000000000 +1000
@@ -1,11 +1,14 @@
- Goals, Design and Implementation of the
- new ultra-scalable O(1) scheduler
+ Goals, Design and Implementation of the ultra-scalable O(1) scheduler by
+ Ingo Molnar and theStaircase Deadline cpu scheduler policy designed by
+ Con Kolivas.
- This is an edited version of an email Ingo Molnar sent to
- lkml on 4 Jan 2002. It describes the goals, design, and
- implementation of Ingo's new ultra-scalable O(1) scheduler.
- Last Updated: 18 April 2002.
+ This was originally an edited version of an email Ingo Molnar sent to
+ lkml on 4 Jan 2002. It describes the goals, design, and implementation
+ of Ingo's ultra-scalable O(1) scheduler. It now contains a description
+ of the Staircase Deadline priority scheduler that was built on this
+ design.
+ Last Updated: Fri, 4 May 2007
Goal
@@ -163,3 +166,222 @@
code is smaller than the old one.
Ingo
+
+
+Staircase Deadline cpu scheduler policy
+================================================
+
+Design summary
+==============
+
+A novel design which incorporates a foreground-background descending priority
+system (the staircase) via a bandwidth allocation matrix according to nice
+level.
+
+
+Features
+========
+
+A starvation free, strict fairness O(1) scalable design with interactivity
+as good as the above restrictions can provide. There is no interactivity
+estimator, no sleep/run measurements and only simple fixed accounting.
+The design has strict enough a design and accounting that task behaviour
+can be modelled and maximum scheduling latencies can be predicted by
+the virtual deadline mechanism that manages runqueues. The prime concern
+in this design is to maintain fairness at all costs determined by nice level,
+yet to maintain as good interactivity as can be allowed within the
+constraints of strict fairness.
+
+
+Design description
+==================
+
+SD works off the principle of providing each task a quota of runtime that it is
+allowed to run at a number of priority levels determined by its static priority
+(ie. its nice level). If the task uses up its quota it has its priority
+decremented to the next level determined by a priority matrix. Once every
+runtime quota has been consumed of every priority level, a task is queued on the
+"expired" array. When no other tasks exist with quota, the expired array is
+activated and fresh quotas are handed out. This is all done in O(1).
+
+Design details
+==============
+
+Each task keeps a record of its own entitlement of cpu time. Most of the rest of
+these details apply to non-realtime tasks as rt task management is straight
+forward.
+
+Each runqueue keeps a record of what major epoch it is up to in the
+rq->prio_rotation field which is incremented on each major epoch. It also
+keeps a record of the current prio_level for each static priority task.
+
+Each task keeps a record of what major runqueue epoch it was last running
+on in p->rotation. It also keeps a record of what priority levels it has
+already been allocated quota from during this epoch in a bitmap p->bitmap.
+
+The only tunable that determines all other details is the RR_INTERVAL. This
+is set to 8ms, and is scaled gently upwards with more cpus. This value is
+tunable via a /proc interface.
+
+All tasks are initially given a quota based on RR_INTERVAL. This is equal to
+RR_INTERVAL between nice values of -6 and 0, half that size above nice 0, and
+progressively larger for nice values from -1 to -20. This is assigned to
+p->quota and only changes with changes in nice level.
+
+As a task is first queued, it checks in recalc_task_prio to see if it has run at
+this runqueue's current priority rotation. If it has not, it will have its
+p->prio level set according to the first slot in a "priority matrix" and will be
+given a p->time_slice equal to the p->quota, and has its allocation bitmap bit
+set in p->bitmap for this prio level. It is then queued on the current active
+priority array.
+
+If a task has already been running during this major epoch, and it has
+p->time_slice left and the rq->prio_quota for the task's p->prio still
+has quota, it will be placed back on the active array, but no more quota
+will be added.
+
+If a task has been running during this major epoch, but does not have
+p->time_slice left, it will find the next lowest priority in its bitmap that it
+has not been allocated quota from. It then gets the a full quota in
+p->time_slice. It is then queued on the current active priority array at the
+newly determined lower priority.
+
+If a task has been running during this major epoch, and does not have
+any entitlement left in p->bitmap and no time_slice left, it will have its
+bitmap cleared, and be queued at its best prio again, but on the expired
+priority array.
+
+When a task is queued, it has its relevant bit set in the array->prio_bitmap.
+
+p->time_slice is stored in nanosconds and is updated via update_cpu_clock on
+schedule() and scheduler_tick. If p->time_slice is below zero then the
+recalc_task_prio is readjusted and the task rescheduled.
+
+
+Priority Matrix
+===============
+
+In order to minimise the latencies between tasks of different nice levels
+running concurrently, the dynamic priority slots where different nice levels
+are queued are dithered instead of being sequential. What this means is that
+there are 40 priority slots where a task may run during one major rotation,
+and the allocation of slots is dependant on nice level. In the
+following table, a zero represents a slot where the task may run.
+
+PRIORITY:0..................20.................39
+nice -20 0000000000000000000000000000000000000000
+nice -10 1000100010001000100010001000100010010000
+nice 0 1010101010101010101010101010101010101010
+nice 5 1011010110110101101101011011010110110110
+nice 10 1110111011101110111011101110111011101110
+nice 15 1111111011111110111111101111111011111110
+nice 19 1111111111111111111111111111111111111110
+
+As can be seen, a nice -20 task runs in every priority slot whereas a nice 19
+task only runs one slot per major rotation. This dithered table allows for the
+smallest possible maximum latencies between tasks of varying nice levels, thus
+allowing vastly different nice levels to be used.
+
+SCHED_BATCH tasks are managed slightly differently, receiving only the top
+slots from its priority bitmap giving it equal cpu as SCHED_NORMAL, but
+slightly higher latencies.
+
+
+Modelling deadline behaviour
+============================
+
+As the accounting in this design is hard and not modified by sleep average
+calculations or interactivity modifiers, it is possible to accurately
+predict the maximum latency that a task may experience under different
+conditions. This is a virtual deadline mechanism enforced by mandatory
+timeslice expiration and not outside bandwidth measurement.
+
+The maximum duration a task can run during one major epoch is determined by its
+nice value. Nice 0 tasks can run at 19 different priority levels for RR_INTERVAL
+duration during each epoch. Nice 10 tasks can run at 9 priority levels for each
+epoch, and so on. The table in the priority matrix above demonstrates how this
+is enforced.
+
+Therefore the maximum duration a runqueue epoch can take is determined by
+the number of tasks running, and their nice level. After that, the maximum
+duration it can take before a task can wait before it get scheduled is
+determined by the position of its first slot on the matrix.
+
+In the following examples, these are _worst case scenarios_ and would rarely
+occur, but can be modelled nonetheless to determine the maximum possible
+latency.
+
+So for example, if two nice 0 tasks are running, and one has just expired as
+another is activated for the first time receiving a full quota for this
+runqueue rotation, the first task will wait:
+
+nr_tasks * max_duration + nice_difference * rr_interval
+1 * 19 * RR_INTERVAL + 0 = 152ms
+
+In the presence of a nice 10 task, a nice 0 task would wait a maximum of
+1 * 10 * RR_INTERVAL + 0 = 80ms
+
+In the presence of a nice 0 task, a nice 10 task would wait a maximum of
+1 * 19 * RR_INTERVAL + 1 * RR_INTERVAL = 160ms
+
+More useful than these values, though, are the average latencies which are
+a matter of determining the average distance between priority slots of
+different nice values and multiplying them by the tasks' quota. For example
+in the presence of a nice -10 task, a nice 0 task will wait either one or
+two slots. Given that nice -10 tasks have a quota 2.5 times the RR_INTERVAL,
+this means the latencies will alternate between 2.5 and 5 RR_INTERVALs or
+20 and 40ms respectively (on uniprocessor at 1000HZ).
+
+
+Achieving interactivity
+=======================
+
+A requirement of this scheduler design was to achieve good interactivity
+despite being a completely fair deadline based design. The disadvantage of
+designs that try to achieve interactivity is that they usually do so at
+the expense of maintaining fairness. As cpu speeds increase, the requirement
+for some sort of metered unfairness towards interactive tasks becomes a less
+desirable phenomenon, but low latency and fairness remains mandatory to
+good interactive performance.
+
+This design relies on the fact that interactive tasks, by their nature,
+sleep often. Most fair scheduling designs end up penalising such tasks
+indirectly giving them less than their fair possible share because of the
+sleep, and have to use a mechanism of bonusing their priority to offset
+this based on the duration they sleep. This becomes increasingly inaccurate
+as the number of running tasks rises and more tasks spend time waiting on
+runqueues rather than sleeping, and it is impossible to tell whether the
+task that's waiting on a runqueue only intends to run for a short period and
+then sleep again after than runqueue wait. Furthermore, all such designs rely
+on a period of time to pass to accumulate some form of statistic on the task
+before deciding on how much to give them preference. The shorter this period,
+the more rapidly bursts of cpu ruin the interactive tasks behaviour. The
+longer this period, the longer it takes for interactive tasks to get low
+scheduling latencies and fair cpu.
+
+This design does not measure sleep time at all. Interactive tasks that sleep
+often will wake up having consumed very little if any of their quota for
+the current major priority rotation. The longer they have slept, the less
+likely they are to even be on the current major priority rotation. Once
+woken up, though, they get to use up a their full quota for that epoch,
+whether part of a quota remains or a full quota. Overall, however, they
+can still only run as much cpu time for that epoch as any other task of the
+same nice level. This means that two tasks behaving completely differently
+from fully cpu bound to waking/sleeping extremely frequently will still
+get the same quota of cpu, but the latter will be using its quota for that
+epoch in bursts rather than continuously. This guarantees that interactive
+tasks get the same amount of cpu as cpu bound ones.
+
+The other requirement of interactive tasks is also to obtain low latencies
+for when they are scheduled. Unlike fully cpu bound tasks and the maximum
+latencies possible described in the modelling deadline behaviour section
+above, tasks that sleep will wake up with quota available usually at the
+current runqueue's priority_level or better. This means that the most latency
+they are likely to see is one RR_INTERVAL, and often they will preempt the
+current task if it is not of a sleeping nature. This then guarantees very
+low latency for interactive tasks, and the lowest latencies for the least
+cpu bound tasks.
+
+
+Fri, 4 May 2007
+Con Kolivas <kernel@kolivas.org>
Index: linux-2.6.22-ck1/Documentation/sysctl/kernel.txt
===================================================================
--- linux-2.6.22-ck1.orig/Documentation/sysctl/kernel.txt 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/Documentation/sysctl/kernel.txt 2007-07-10 14:55:20.000000000 +1000
@@ -25,6 +25,9 @@
- domainname
- hostname
- hotplug
+- interactive
+- iso_cpu
+- iso_period
- java-appletviewer [ binfmt_java, obsolete ]
- java-interpreter [ binfmt_java, obsolete ]
- kstack_depth_to_print [ X86 only ]
@@ -43,6 +46,7 @@
- printk
- real-root-dev ==> Documentation/initrd.txt
- reboot-cmd [ SPARC only ]
+- rr_interval
- rtsig-max
- rtsig-nr
- sem
@@ -164,6 +168,40 @@
==============================================================
+interactive:
+
+The staircase-deadline cpu scheduler can be set in either purely
+forward-looking mode for absolutely rigid fairness and cpu distribution
+according to nice level, or it can allow a small per-process history
+to smooth out cpu usage perturbations common in interactive tasks by
+enabling this sysctl. While small fairness issues can arise with this
+enabled, overall fairness is usually still strongly maintained and
+starvation is never possible. Enabling this can significantly smooth
+out 3d graphics and games.
+
+Default value is 1 (enabled).
+
+==============================================================
+
+iso_cpu:
+
+This sets the percentage cpu that the unprivileged SCHED_ISO tasks can
+run effectively at realtime priority, averaged over a rolling iso_period
+seconds.
+
+Set to 80 (percent) by default.
+
+==============================================================
+
+iso_period:
+
+This sets the number of seconds over which SCHED_ISO cpu usage is averaged
+to see if it exceeds its allocated cpu bandwidth.
+
+Set to 5 (seconds) by default.
+
+==============================================================
+
l2cr: (PPC only)
This flag controls the L2 cache of G3 processor boards. If
@@ -288,6 +326,19 @@
==============================================================
+rr_interval:
+
+This is the smallest duration that any cpu process scheduling unit
+will run for. Increasing this value can increase throughput of cpu
+bound tasks substantially but at the expense of increased latencies
+overall. This value is in milliseconds and the default value chosen
+depends on the number of cpus available at scheduler initialisation
+with a minimum of 8.
+
+Valid values are from 1-5000.
+
+==============================================================
+
rtsig-max & rtsig-nr:
The file rtsig-max can be used to tune the maximum number
Index: linux-2.6.22-ck1/fs/pipe.c
===================================================================
--- linux-2.6.22-ck1.orig/fs/pipe.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/fs/pipe.c 2007-07-10 14:55:02.000000000 +1000
@@ -41,12 +41,7 @@
{
DEFINE_WAIT(wait);
- /*
- * Pipes are system-local resources, so sleeping on them
- * is considered a noninteractive wait:
- */
- prepare_to_wait(&pipe->wait, &wait,
- TASK_INTERRUPTIBLE | TASK_NONINTERACTIVE);
+ prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE);
if (pipe->inode)
mutex_unlock(&pipe->inode->i_mutex);
schedule();
Index: linux-2.6.22-ck1/fs/proc/array.c
===================================================================
--- linux-2.6.22-ck1.orig/fs/proc/array.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/fs/proc/array.c 2007-07-10 14:55:02.000000000 +1000
@@ -165,7 +165,6 @@
rcu_read_lock();
buffer += sprintf(buffer,
"State:\t%s\n"
- "SleepAVG:\t%lu%%\n"
"Tgid:\t%d\n"
"Pid:\t%d\n"
"PPid:\t%d\n"
@@ -173,7 +172,6 @@
"Uid:\t%d\t%d\t%d\t%d\n"
"Gid:\t%d\t%d\t%d\t%d\n",
get_task_state(p),
- (p->sleep_avg/1024)*100/(1020000000/1024),
p->tgid, p->pid,
pid_alive(p) ? rcu_dereference(p->real_parent)->tgid : 0,
pid_alive(p) && p->ptrace ? rcu_dereference(p->parent)->pid : 0,
Index: linux-2.6.22-ck1/include/linux/init_task.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/init_task.h 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/init_task.h 2007-07-10 14:55:20.000000000 +1000
@@ -125,13 +125,15 @@
.prio = MAX_PRIO-20, \
.static_prio = MAX_PRIO-20, \
.normal_prio = MAX_PRIO-20, \
+ .rotation = 0, \
.policy = SCHED_NORMAL, \
.cpus_allowed = CPU_MASK_ALL, \
.mm = NULL, \
.active_mm = &init_mm, \
.run_list = LIST_HEAD_INIT(tsk.run_list), \
.ioprio = 0, \
- .time_slice = HZ, \
+ .time_slice = 1000000000, \
+ .quota = 1000000000, \
.tasks = LIST_HEAD_INIT(tsk.tasks), \
.ptrace_children= LIST_HEAD_INIT(tsk.ptrace_children), \
.ptrace_list = LIST_HEAD_INIT(tsk.ptrace_list), \
@@ -158,6 +160,7 @@
.signal = {{0}}}, \
.blocked = {{0}}, \
.alloc_lock = __SPIN_LOCK_UNLOCKED(tsk.alloc_lock), \
+ .mutexes_held = 0, \
.journal_info = NULL, \
.cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \
.fs_excl = ATOMIC_INIT(0), \
Index: linux-2.6.22-ck1/kernel/softirq.c
===================================================================
--- linux-2.6.22-ck1.orig/kernel/softirq.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/kernel/softirq.c 2007-07-10 14:55:02.000000000 +1000
@@ -488,7 +488,7 @@
static int ksoftirqd(void * __bind_cpu)
{
- set_user_nice(current, 19);
+ set_user_nice(current, 15);
current->flags |= PF_NOFREEZE;
set_current_state(TASK_INTERRUPTIBLE);
Index: linux-2.6.22-ck1/kernel/workqueue.c
===================================================================
--- linux-2.6.22-ck1.orig/kernel/workqueue.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/kernel/workqueue.c 2007-07-10 14:55:02.000000000 +1000
@@ -285,8 +285,6 @@
if (!cwq->wq->freezeable)
current->flags |= PF_NOFREEZE;
- set_user_nice(current, -5);
-
for (;;) {
prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
if (!freezing(current) &&
Index: linux-2.6.22-ck1/kernel/kthread.c
===================================================================
--- linux-2.6.22-ck1.orig/kernel/kthread.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/kernel/kthread.c 2007-07-10 14:55:02.000000000 +1000
@@ -223,7 +223,6 @@
ignore_signals(tsk);
- set_user_nice(tsk, -5);
set_cpus_allowed(tsk, CPU_MASK_ALL);
}
Index: linux-2.6.22-ck1/kernel/fork.c
===================================================================
--- linux-2.6.22-ck1.orig/kernel/fork.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/kernel/fork.c 2007-07-10 14:55:20.000000000 +1000
@@ -1063,6 +1063,7 @@
p->io_context = NULL;
p->io_wait = NULL;
p->audit_context = NULL;
+ p->mutexes_held = 0;
cpuset_fork(p);
#ifdef CONFIG_NUMA
p->mempolicy = mpol_copy(p->mempolicy);
Index: linux-2.6.22-ck1/kernel/mutex.c
===================================================================
--- linux-2.6.22-ck1.orig/kernel/mutex.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/kernel/mutex.c 2007-07-10 14:55:20.000000000 +1000
@@ -60,6 +60,16 @@
static void fastcall noinline __sched
__mutex_lock_slowpath(atomic_t *lock_count);
+static inline void inc_mutex_count(void)
+{
+ current->mutexes_held++;
+}
+
+static inline void dec_mutex_count(void)
+{
+ current->mutexes_held--;
+}
+
/***
* mutex_lock - acquire the mutex
* @lock: the mutex to be acquired
@@ -89,6 +99,7 @@
* 'unlocked' into 'locked' state.
*/
__mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
+ inc_mutex_count();
}
EXPORT_SYMBOL(mutex_lock);
@@ -114,6 +125,7 @@
* into 'unlocked' state:
*/
__mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
+ dec_mutex_count();
}
EXPORT_SYMBOL(mutex_unlock);
@@ -283,9 +295,14 @@
*/
int fastcall __sched mutex_lock_interruptible(struct mutex *lock)
{
+ int ret;
+
might_sleep();
- return __mutex_fastpath_lock_retval
+ ret = __mutex_fastpath_lock_retval
(&lock->count, __mutex_lock_interruptible_slowpath);
+ if (likely(!ret))
+ inc_mutex_count();
+ return ret;
}
EXPORT_SYMBOL(mutex_lock_interruptible);
@@ -340,8 +357,12 @@
*/
int fastcall __sched mutex_trylock(struct mutex *lock)
{
- return __mutex_fastpath_trylock(&lock->count,
+ int ret = __mutex_fastpath_trylock(&lock->count,
__mutex_trylock_slowpath);
+
+ if (likely(ret))
+ inc_mutex_count();
+ return ret;
}
EXPORT_SYMBOL(mutex_trylock);
Index: linux-2.6.22-ck1/block/cfq-iosched.c
===================================================================
--- linux-2.6.22-ck1.orig/block/cfq-iosched.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/block/cfq-iosched.c 2007-07-10 14:55:21.000000000 +1000
@@ -1276,10 +1276,12 @@
printk(KERN_ERR "cfq: bad prio %x\n", ioprio_class);
case IOPRIO_CLASS_NONE:
/*
- * no prio set, place us in the middle of the BE classes
+ * Select class and ioprio according to policy and nice
*/
+ cfqq->ioprio_class = task_policy_ioprio_class(tsk);
cfqq->ioprio = task_nice_ioprio(tsk);
- cfqq->ioprio_class = IOPRIO_CLASS_BE;
+ if (cfqq->ioprio_class == IOPRIO_CLASS_IDLE)
+ cfq_clear_cfqq_idle_window(cfqq);
break;
case IOPRIO_CLASS_RT:
cfqq->ioprio = task_ioprio(tsk);
Index: linux-2.6.22-ck1/include/linux/ioprio.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/ioprio.h 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/ioprio.h 2007-07-10 14:55:21.000000000 +1000
@@ -22,7 +22,7 @@
* class, the default for any process. IDLE is the idle scheduling class, it
* is only served when no one else is using the disk.
*/
-enum {
+enum ioprio_class {
IOPRIO_CLASS_NONE,
IOPRIO_CLASS_RT,
IOPRIO_CLASS_BE,
@@ -51,8 +51,25 @@
return IOPRIO_PRIO_DATA(task->ioprio);
}
+static inline enum ioprio_class
+ task_policy_ioprio_class(struct task_struct *task)
+{
+ if (rt_task(task))
+ return IOPRIO_CLASS_RT;
+ if (idleprio_task(task))
+ return IOPRIO_CLASS_IDLE;
+ return IOPRIO_CLASS_BE;
+}
+
static inline int task_nice_ioprio(struct task_struct *task)
{
+ if (rt_task(task))
+ return (MAX_RT_PRIO - task->rt_priority) * IOPRIO_BE_NR /
+ (MAX_RT_PRIO + 1);
+ if (iso_task(task))
+ return 0;
+ if (idleprio_task(task))
+ return IOPRIO_BE_NR - 1;
return (task_nice(task) + 20) / 5;
}
Index: linux-2.6.22-ck1/Documentation/sysctl/vm.txt
===================================================================
--- linux-2.6.22-ck1.orig/Documentation/sysctl/vm.txt 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/Documentation/sysctl/vm.txt 2007-07-10 14:55:23.000000000 +1000
@@ -22,6 +22,8 @@
- dirty_background_ratio
- dirty_expire_centisecs
- dirty_writeback_centisecs
+- hardmaplimit
+- mapped
- max_map_count
- min_free_kbytes
- laptop_mode
@@ -31,12 +33,15 @@
- min_unmapped_ratio
- min_slab_ratio
- panic_on_oom
+- swap_prefetch
+- swap_prefetch_delay
+- swap_prefetch_sleep
==============================================================
dirty_ratio, dirty_background_ratio, dirty_expire_centisecs,
dirty_writeback_centisecs, vfs_cache_pressure, laptop_mode,
-block_dump, swap_token_timeout, drop-caches:
+block_dump, swap_token_timeout, drop-caches, tail_largefiles:
See Documentation/filesystems/proc.txt
@@ -86,6 +91,27 @@
==============================================================
+hardmaplimit:
+
+This flag makes the vm adhere to the mapped value as closely as possible
+except in the most extreme vm stress where doing so would provoke an out
+of memory condition (see mapped below).
+
+Enabled by default.
+
+==============================================================
+
+mapped:
+
+This is the percentage ram that is filled with mapped pages (applications)
+before the vm will start reclaiming mapped pages by moving them to swap.
+It is altered by the relative stress of the vm at the time so is not
+strictly adhered to to prevent provoking out of memory kills.
+
+Set to 66 by default.
+
+==============================================================
+
max_map_count:
This file contains the maximum number of memory map areas a process
@@ -216,3 +242,37 @@
The default value is 0.
1 and 2 are for failover of clustering. Please select either
according to your policy of failover.
+
+==============================================================
+
+swap_prefetch
+
+This enables or disables the swap prefetching feature. When the virtual
+memory subsystem has been extremely idle for at least swap_prefetch_sleep
+seconds it will start copying back pages from swap into the swapcache and keep
+a copy in swap. Valid values are 0 - 3. A value of 0 disables swap
+prefetching, 1 enables it unless laptop_mode is enabled, 2 enables it in the
+presence of laptop_mode, and 3 enables it unconditionally, ignoring whether
+the system is idle or not. If set to 0, swap prefetch wil not even try to keep
+record of ram swapped out to have the most minimal impact on performance.
+
+The default value is 1.
+
+==============================================================
+
+swap_prefetch_delay
+
+This is the time in seconds that swap prefetching is delayed upon finding
+the system is not idle (ie the vm is busy or non-niced cpu load is present).
+
+The default value is 1.
+
+==============================================================
+
+swap_prefetch_sleep
+
+This is the time in seconds that the swap prefetch kernel thread is put to
+sleep for when the ram is found to be full and it is unable to prefetch
+further.
+
+The default value is 5.
Index: linux-2.6.22-ck1/include/linux/swap.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/swap.h 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/swap.h 2007-07-10 14:55:22.000000000 +1000
@@ -180,6 +180,7 @@
/* linux/mm/swap.c */
extern void FASTCALL(lru_cache_add(struct page *));
extern void FASTCALL(lru_cache_add_active(struct page *));
+extern void FASTCALL(lru_cache_add_tail(struct page *));
extern void FASTCALL(activate_page(struct page *));
extern void FASTCALL(mark_page_accessed(struct page *));
extern void lru_add_drain(void);
@@ -188,9 +189,11 @@
extern void swap_setup(void);
/* linux/mm/vmscan.c */
-extern unsigned long try_to_free_pages(struct zone **, gfp_t);
+extern unsigned long try_to_free_pages(struct zone **, gfp_t,
+ struct task_struct *p);
extern unsigned long shrink_all_memory(unsigned long nr_pages);
-extern int vm_swappiness;
+extern int vm_mapped;
+extern int vm_hardmaplimit;
extern int remove_mapping(struct address_space *mapping, struct page *page);
extern long vm_total_pages;
@@ -237,6 +240,7 @@
extern struct page * lookup_swap_cache(swp_entry_t);
extern struct page * read_swap_cache_async(swp_entry_t, struct vm_area_struct *vma,
unsigned long addr);
+extern int add_to_swap_cache(struct page *page, swp_entry_t entry);
/* linux/mm/swapfile.c */
extern long total_swap_pages;
extern unsigned int nr_swapfiles;
Index: linux-2.6.22-ck1/init/Kconfig
===================================================================
--- linux-2.6.22-ck1.orig/init/Kconfig 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/init/Kconfig 2007-07-10 14:55:22.000000000 +1000
@@ -105,6 +105,28 @@
used to provide more virtual memory than the actual RAM present
in your computer. If unsure say Y.
+config SWAP_PREFETCH
+ bool "Support for prefetching swapped memory"
+ depends on SWAP
+ default y
+ ---help---
+ This option will allow the kernel to prefetch swapped memory pages
+ when idle. The pages will be kept on both swap and in swap_cache
+ thus avoiding the need for further I/O if either ram or swap space
+ is required.
+
+ What this will do on workstations is slowly bring back applications
+ that have swapped out after memory intensive workloads back into
+ physical ram if you have free ram at a later stage and the machine
+ is relatively idle. This means that when you come back to your
+ computer after leaving it idle for a while, applications will come
+ to life faster. Note that your swap usage will appear to increase
+ but these are cached pages, can be dropped freely by the vm, and it
+ should stabilise around 50% swap usage maximum.
+
+ Workstations and multiuser workstation servers will most likely want
+ to say Y.
+
config SYSVIPC
bool "System V IPC"
---help---
Index: linux-2.6.22-ck1/mm/Makefile
===================================================================
--- linux-2.6.22-ck1.orig/mm/Makefile 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/mm/Makefile 2007-07-10 14:55:22.000000000 +1000
@@ -17,6 +17,7 @@
obj-y += bounce.o
endif
obj-$(CONFIG_SWAP) += page_io.o swap_state.o swapfile.o thrash.o
+obj-$(CONFIG_SWAP_PREFETCH) += swap_prefetch.o
obj-$(CONFIG_HUGETLBFS) += hugetlb.o
obj-$(CONFIG_NUMA) += mempolicy.o
obj-$(CONFIG_SPARSEMEM) += sparse.o
Index: linux-2.6.22-ck1/mm/swap.c
===================================================================
--- linux-2.6.22-ck1.orig/mm/swap.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/mm/swap.c 2007-07-10 14:55:23.000000000 +1000
@@ -17,6 +17,7 @@
#include <linux/sched.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
+#include <linux/swap-prefetch.h>
#include <linux/mman.h>
#include <linux/pagemap.h>
#include <linux/pagevec.h>
@@ -176,6 +177,7 @@
*/
static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, };
static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, };
+static DEFINE_PER_CPU(struct pagevec, lru_add_tail_pvecs) = { 0, };
void fastcall lru_cache_add(struct page *page)
{
@@ -197,6 +199,31 @@
put_cpu_var(lru_add_active_pvecs);
}
+static void __pagevec_lru_add_tail(struct pagevec *pvec)
+{
+ int i;
+ struct zone *zone = NULL;
+
+ for (i = 0; i < pagevec_count(pvec); i++) {
+ struct page *page = pvec->pages[i];
+ struct zone *pagezone = page_zone(page);
+
+ if (pagezone != zone) {
+ if (zone)
+ spin_unlock_irq(&zone->lru_lock);
+ zone = pagezone;
+ spin_lock_irq(&zone->lru_lock);
+ }
+ BUG_ON(PageLRU(page));
+ SetPageLRU(page);
+ add_page_to_inactive_list_tail(zone, page);
+ }
+ if (zone)
+ spin_unlock_irq(&zone->lru_lock);
+ release_pages(pvec->pages, pvec->nr, pvec->cold);
+ pagevec_reinit(pvec);
+}
+
static void __lru_add_drain(int cpu)
{
struct pagevec *pvec = &per_cpu(lru_add_pvecs, cpu);
@@ -207,6 +234,9 @@
pvec = &per_cpu(lru_add_active_pvecs, cpu);
if (pagevec_count(pvec))
__pagevec_lru_add_active(pvec);
+ pvec = &per_cpu(lru_add_tail_pvecs, cpu);
+ if (pagevec_count(pvec))
+ __pagevec_lru_add_tail(pvec);
}
void lru_add_drain(void)
@@ -403,6 +433,20 @@
}
/*
+ * Function used uniquely to put pages back to the lru at the end of the
+ * inactive list to preserve the lru order.
+ */
+void fastcall lru_cache_add_tail(struct page *page)
+{
+ struct pagevec *pvec = &get_cpu_var(lru_add_tail_pvecs);
+
+ page_cache_get(page);
+ if (!pagevec_add(pvec, page))
+ __pagevec_lru_add_tail(pvec);
+ put_cpu_var(lru_add_pvecs);
+}
+
+/*
* Try to drop buffers from the pages in a pagevec
*/
void pagevec_strip(struct pagevec *pvec)
@@ -514,6 +558,9 @@
* Right now other parts of the system means that we
* _really_ don't want to cluster much more
*/
+
+ prepare_swap_prefetch();
+
#ifdef CONFIG_HOTPLUG_CPU
hotcpu_notifier(cpu_swap_callback, 0);
#endif
Index: linux-2.6.22-ck1/mm/swap_prefetch.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.22-ck1/mm/swap_prefetch.c 2007-07-10 14:55:22.000000000 +1000
@@ -0,0 +1,542 @@
+/*
+ * linux/mm/swap_prefetch.c
+ *
+ * Copyright (C) 2005-2007 Con Kolivas
+ *
+ * Written by Con Kolivas <kernel@kolivas.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/swap.h>
+#include <linux/swap-prefetch.h>
+#include <linux/ioprio.h>
+#include <linux/kthread.h>
+#include <linux/pagemap.h>
+#include <linux/syscalls.h>
+#include <linux/writeback.h>
+#include <linux/vmstat.h>
+#include <linux/freezer.h>
+
+/*
+ * sysctls:
+ * swap_prefetch: 0. Disable swap prefetching
+ * 1. Prefetch only when idle and not with laptop_mode
+ * 2. Prefetch when idle and with laptop_mode
+ * 3. Prefetch at all times.
+ * swap_prefetch_delay: Number of seconds to delay prefetching when system
+ * is not idle.
+ * swap_prefetch_sleep: Number of seconds to put kprefetchd to sleep when
+ * unable to prefetch.
+ */
+int swap_prefetch __read_mostly = 1;
+int swap_prefetch_delay __read_mostly = 1;
+int swap_prefetch_sleep __read_mostly = 5;
+
+#define PREFETCH_DELAY (HZ * swap_prefetch_delay)
+#define PREFETCH_SLEEP ((HZ * swap_prefetch_sleep) ? : 1)
+
+struct swapped_root {
+ unsigned long busy; /* vm busy */
+ spinlock_t lock; /* protects all data */
+ struct list_head list; /* MRU list of swapped pages */
+ struct radix_tree_root swap_tree; /* Lookup tree of pages */
+ unsigned int count; /* Number of entries */
+ unsigned int maxcount; /* Maximum entries allowed */
+ struct kmem_cache *cache; /* Of struct swapped_entry */
+};
+
+static struct swapped_root swapped = {
+ .lock = SPIN_LOCK_UNLOCKED,
+ .list = LIST_HEAD_INIT(swapped.list),
+ .swap_tree = RADIX_TREE_INIT(GFP_ATOMIC),
+};
+
+static struct task_struct *kprefetchd_task;
+
+/*
+ * We check to see no part of the vm is busy. If it is this will interrupt
+ * trickle_swap and wait another PREFETCH_DELAY. Purposefully racy.
+ */
+inline void delay_swap_prefetch(void)
+{
+ if (!test_bit(0, &swapped.busy))
+ __set_bit(0, &swapped.busy);
+}
+
+/*
+ * If laptop_mode is enabled don't prefetch to avoid hard drives
+ * doing unnecessary spin-ups unless swap_prefetch is explicitly
+ * set to a higher value.
+ */
+static inline int prefetch_enabled(void)
+{
+ if (swap_prefetch <= laptop_mode)
+ return 0;
+ return 1;
+}
+
+static int kprefetchd_awake;
+
+/*
+ * Drop behind accounting which keeps a list of the most recently used swap
+ * entries. Entries are removed lazily by kprefetchd.
+ */
+void add_to_swapped_list(struct page *page)
+{
+ struct swapped_entry *entry;
+ unsigned long index, flags;
+
+ if (!prefetch_enabled())
+ goto out;
+
+ spin_lock_irqsave(&swapped.lock, flags);
+ if (swapped.count >= swapped.maxcount) {
+ /*
+ * Once the number of entries exceeds maxcount we start
+ * removing the least recently used entries.
+ */
+ entry = list_entry(swapped.list.next,
+ struct swapped_entry, swapped_list);
+ radix_tree_delete(&swapped.swap_tree, entry->swp_entry.val);
+ list_del(&entry->swapped_list);
+ swapped.count--;
+ } else {
+ entry = kmem_cache_alloc(swapped.cache, GFP_ATOMIC);
+ if (unlikely(!entry))
+ /* bad, can't allocate more mem */
+ goto out_locked;
+ }
+
+ index = page_private(page);
+ entry->swp_entry.val = index;
+ /*
+ * On numa we need to store the node id to ensure that we prefetch to
+ * the same node it came from.
+ */
+ store_swap_entry_node(entry, page);
+
+ if (likely(!radix_tree_insert(&swapped.swap_tree, index, entry))) {
+ list_add(&entry->swapped_list, &swapped.list);
+ swapped.count++;
+ } else
+ kmem_cache_free(swapped.cache, entry);
+
+out_locked:
+ spin_unlock_irqrestore(&swapped.lock, flags);
+out:
+ if (!kprefetchd_awake)
+ wake_up_process(kprefetchd_task);
+ return;
+}
+
+/*
+ * Removes entries from the swapped_list. The radix tree allows us to quickly
+ * look up the entry from the index without having to iterate over the whole
+ * list.
+ */
+static void remove_from_swapped_list(const unsigned long index)
+{
+ struct swapped_entry *entry;
+ unsigned long flags;
+
+ spin_lock_irqsave(&swapped.lock, flags);
+ entry = radix_tree_delete(&swapped.swap_tree, index);
+ if (likely(entry)) {
+ list_del(&entry->swapped_list);
+ swapped.count--;
+ kmem_cache_free(swapped.cache, entry);
+ }
+ spin_unlock_irqrestore(&swapped.lock, flags);
+}
+
+enum trickle_return {
+ TRICKLE_SUCCESS,
+ TRICKLE_FAILED,
+ TRICKLE_DELAY,
+};
+
+struct node_stats {
+ /* Free ram after a cycle of prefetching */
+ unsigned long last_free;
+ /* Free ram on this cycle of checking prefetch_suitable */
+ unsigned long current_free;
+ /* The amount of free ram before we start prefetching */
+ unsigned long highfree[MAX_NR_ZONES];
+ /* The amount of free ram where we will stop prefetching */
+ unsigned long lowfree[MAX_NR_ZONES];
+ /* highfree or lowfree depending on whether we've hit a watermark */
+ unsigned long *pointfree[MAX_NR_ZONES];
+};
+
+/*
+ * prefetch_stats stores the free ram data of each node and this is used to
+ * determine if a node is suitable for prefetching into.
+ */
+struct prefetch_stats {
+ /* Which nodes are currently suited to prefetching */
+ nodemask_t prefetch_nodes;
+ /* Total pages we've prefetched on this wakeup of kprefetchd */
+ unsigned long prefetched_pages;
+ struct node_stats node[MAX_NUMNODES];
+};
+
+static struct prefetch_stats sp_stat;
+
+/*
+ * This tries to read a swp_entry_t into swap cache for swap prefetching.
+ * If it returns TRICKLE_DELAY we should delay further prefetching.
+ */
+static enum trickle_return trickle_swap_cache_async(const swp_entry_t entry,
+ const int node)
+{
+ enum trickle_return ret = TRICKLE_FAILED;
+ unsigned long flags;
+ struct page *page;
+
+ read_lock_irqsave(&swapper_space.tree_lock, flags);
+ /* Entry may already exist */
+ page = radix_tree_lookup(&swapper_space.page_tree, entry.val);
+ read_unlock_irqrestore(&swapper_space.tree_lock, flags);
+ if (page)
+ goto out;
+
+ /*
+ * Get a new page to read from swap. We have already checked the
+ * watermarks so __alloc_pages will not call on reclaim.
+ */
+ page = alloc_pages_node(node, GFP_HIGHUSER & ~__GFP_WAIT, 0);
+ if (unlikely(!page)) {
+ ret = TRICKLE_DELAY;
+ goto out;
+ }
+
+ if (add_to_swap_cache(page, entry)) {
+ /* Failed to add to swap cache */
+ goto out_release;
+ }
+
+ /* Add them to the tail of the inactive list to preserve LRU order */
+ lru_cache_add_tail(page);
+ if (unlikely(swap_readpage(NULL, page)))
+ goto out_release;
+
+ sp_stat.prefetched_pages++;
+ sp_stat.node[node].last_free--;
+
+ ret = TRICKLE_SUCCESS;
+out_release:
+ page_cache_release(page);
+out:
+ /*
+ * All entries are removed here lazily. This avoids the cost of
+ * remove_from_swapped_list during normal swapin. Thus there are
+ * usually many stale entries.
+ */
+ remove_from_swapped_list(entry.val);
+ return ret;
+}
+
+static void clear_last_prefetch_free(void)
+{
+ int node;
+
+ /*
+ * Reset the nodes suitable for prefetching to all nodes. We could
+ * update the data to take into account memory hotplug if desired..
+ */
+ sp_stat.prefetch_nodes = node_online_map;
+ for_each_node_mask(node, sp_stat.prefetch_nodes) {
+ struct node_stats *ns = &sp_stat.node[node];
+
+ ns->last_free = 0;
+ }
+}
+
+static void clear_current_prefetch_free(void)
+{
+ int node;
+
+ sp_stat.prefetch_nodes = node_online_map;
+ for_each_node_mask(node, sp_stat.prefetch_nodes) {
+ struct node_stats *ns = &sp_stat.node[node];
+
+ ns->current_free = 0;
+ }
+}
+
+/*
+ * This updates the high and low watermarks of amount of free ram in each
+ * node used to start and stop prefetching. We prefetch from pages_high * 4
+ * down to pages_high * 3.
+ */
+static void examine_free_limits(void)
+{
+ struct zone *z;
+
+ for_each_zone(z) {
+ struct node_stats *ns;
+ int idx;
+
+ if (!populated_zone(z))
+ continue;
+
+ ns = &sp_stat.node[zone_to_nid(z)];
+ idx = zone_idx(z);
+ ns->lowfree[idx] = z->pages_high * 3;
+ ns->highfree[idx] = ns->lowfree[idx] + z->pages_high;
+
+ if (zone_page_state(z, NR_FREE_PAGES) > ns->highfree[idx]) {
+ /*
+ * We've gotten above the high watermark of free pages
+ * so we can start prefetching till we get to the low
+ * watermark.
+ */
+ ns->pointfree[idx] = &ns->lowfree[idx];
+ }
+ }
+}
+
+/*
+ * We want to be absolutely certain it's ok to start prefetching.
+ */
+static enum trickle_return prefetch_suitable(void)
+{
+ enum trickle_return ret = TRICKLE_DELAY;
+ struct zone *z;
+ int node;
+
+ /*
+ * If swap_prefetch is set to a high value we can ignore load
+ * and prefetch whenever we can. Otherwise we test for vm and
+ * cpu activity.
+ */
+ if (swap_prefetch < 3) {
+ /* Purposefully racy, may return false positive */
+ if (test_bit(0, &swapped.busy)) {
+ __clear_bit(0, &swapped.busy);
+ goto out;
+ }
+
+ /*
+ * above_background_load is expensive so we only perform it
+ * every SWAP_CLUSTER_MAX prefetched_pages.
+ * We test to see if we're above_background_load as disk
+ * activity even at low priority can cause interrupt induced
+ * scheduling latencies.
+ */
+ if (!(sp_stat.prefetched_pages % SWAP_CLUSTER_MAX) &&
+ above_background_load())
+ goto out;
+ }
+ clear_current_prefetch_free();
+
+ /*
+ * Have some hysteresis between where page reclaiming and prefetching
+ * will occur to prevent ping-ponging between them.
+ */
+ for_each_zone(z) {
+ struct node_stats *ns;
+ unsigned long free;
+ int idx;
+
+ if (!populated_zone(z))
+ continue;
+
+ node = zone_to_nid(z);
+ ns = &sp_stat.node[node];
+ idx = zone_idx(z);
+
+ free = zone_page_state(z, NR_FREE_PAGES);
+ if (free < *ns->pointfree[idx]) {
+ /*
+ * Free pages have dropped below the low watermark so
+ * we won't start prefetching again till we hit the
+ * high watermark of free pages.
+ */
+ ns->pointfree[idx] = &ns->highfree[idx];
+ node_clear(node, sp_stat.prefetch_nodes);
+ continue;
+ }
+ ns->current_free += free;
+ }
+
+ /*
+ * We iterate over each node testing to see if it is suitable for
+ * prefetching and clear the nodemask if it is not.
+ */
+ for_each_node_mask(node, sp_stat.prefetch_nodes) {
+ struct node_stats *ns = &sp_stat.node[node];
+
+ /*
+ * We check to see that pages are not being allocated
+ * elsewhere at any significant rate implying any
+ * degree of memory pressure (eg during file reads)
+ */
+ if (ns->last_free) {
+ if (ns->current_free + SWAP_CLUSTER_MAX <
+ ns->last_free) {
+ ns->last_free = ns->current_free;
+ node_clear(node,
+ sp_stat.prefetch_nodes);
+ continue;
+ }
+ } else
+ ns->last_free = ns->current_free;
+
+ /* We shouldn't prefetch when we are doing writeback */
+ if (node_page_state(node, NR_WRITEBACK))
+ node_clear(node, sp_stat.prefetch_nodes);
+ }
+
+ /* Nothing suitable, put kprefetchd back to sleep */
+ if (nodes_empty(sp_stat.prefetch_nodes))
+ return TRICKLE_FAILED;
+
+ /* Survived all that? Hooray we can prefetch! */
+ ret = TRICKLE_SUCCESS;
+out:
+ return ret;
+}
+
+/*
+ * trickle_swap is the main function that initiates the swap prefetching. It
+ * first checks to see if the busy flag is set, and does not prefetch if it
+ * is, as the flag implied we are low on memory or swapping in currently.
+ * Otherwise it runs until prefetch_suitable fails which occurs when the
+ * vm is busy, we prefetch to the watermark, the list is empty or we have
+ * iterated over all entries once.
+ */
+static enum trickle_return trickle_swap(void)
+{
+ enum trickle_return suitable, ret = TRICKLE_DELAY;
+ struct swapped_entry *pos, *n;
+ unsigned long flags;
+
+ if (!prefetch_enabled())
+ return ret;
+
+ examine_free_limits();
+ suitable = prefetch_suitable();
+ if (suitable != TRICKLE_SUCCESS)
+ return suitable;
+ if (list_empty(&swapped.list)) {
+ kprefetchd_awake = 0;
+ return TRICKLE_FAILED;
+ }
+
+ spin_lock_irqsave(&swapped.lock, flags);
+ list_for_each_entry_safe_reverse(pos, n, &swapped.list, swapped_list) {
+ swp_entry_t swp_entry;
+ int node;
+
+ spin_unlock_irqrestore(&swapped.lock, flags);
+ cond_resched();
+ suitable = prefetch_suitable();
+ if (suitable != TRICKLE_SUCCESS) {
+ ret = suitable;
+ goto out_unlocked;
+ }
+
+ spin_lock_irqsave(&swapped.lock, flags);
+ if (unlikely(!pos))
+ continue;
+ node = get_swap_entry_node(pos);
+ if (!node_isset(node, sp_stat.prefetch_nodes)) {
+ /*
+ * We found an entry that belongs to a node that is
+ * not suitable for prefetching so skip it.
+ */
+ continue;
+ }
+ swp_entry = pos->swp_entry;
+ spin_unlock_irqrestore(&swapped.lock, flags);
+
+ if (trickle_swap_cache_async(swp_entry, node) == TRICKLE_DELAY)
+ goto out_unlocked;
+ spin_lock_irqsave(&swapped.lock, flags);
+ }
+ spin_unlock_irqrestore(&swapped.lock, flags);
+
+out_unlocked:
+ if (sp_stat.prefetched_pages) {
+ lru_add_drain();
+ sp_stat.prefetched_pages = 0;
+ }
+ return ret;
+}
+
+static int kprefetchd(void *__unused)
+{
+ struct sched_param param = { .sched_priority = 0 };
+
+ sched_setscheduler(current, SCHED_BATCH, &param);
+ set_user_nice(current, 19);
+ /* Set ioprio to lowest if supported by i/o scheduler */
+ sys_ioprio_set(IOPRIO_WHO_PROCESS, IOPRIO_BE_NR - 1, IOPRIO_CLASS_BE);
+
+ while (!kthread_should_stop()) {
+ try_to_freeze();
+
+ if (!kprefetchd_awake) {
+ set_current_state(TASK_INTERRUPTIBLE);
+ schedule();
+ kprefetchd_awake = 1;
+ }
+
+ if (trickle_swap() == TRICKLE_FAILED)
+ schedule_timeout_interruptible(PREFETCH_SLEEP);
+ else
+ schedule_timeout_interruptible(PREFETCH_DELAY);
+ clear_last_prefetch_free();
+ }
+ return 0;
+}
+
+/*
+ * Create kmem cache for swapped entries
+ */
+void __init prepare_swap_prefetch(void)
+{
+ struct zone *zone;
+
+ swapped.cache = kmem_cache_create("swapped_entry",
+ sizeof(struct swapped_entry), 0, SLAB_PANIC, NULL, NULL);
+
+ /*
+ * We set the limit to more entries than the physical ram.
+ * We remove entries lazily so we need some headroom.
+ */
+ swapped.maxcount = nr_free_pagecache_pages() * 2;
+
+ for_each_zone(zone) {
+ struct node_stats *ns;
+ int idx;
+
+ if (!populated_zone(zone))
+ continue;
+
+ ns = &sp_stat.node[zone_to_nid(zone)];
+ idx = zone_idx(zone);
+ ns->pointfree[idx] = &ns->highfree[idx];
+ }
+}
+
+static int __init kprefetchd_init(void)
+{
+ kprefetchd_task = kthread_run(kprefetchd, NULL, "kprefetchd");
+
+ return 0;
+}
+
+static void __exit kprefetchd_exit(void)
+{
+ kthread_stop(kprefetchd_task);
+}
+
+module_init(kprefetchd_init);
+module_exit(kprefetchd_exit);
Index: linux-2.6.22-ck1/mm/swap_state.c
===================================================================
--- linux-2.6.22-ck1.orig/mm/swap_state.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/mm/swap_state.c 2007-07-10 14:55:22.000000000 +1000
@@ -10,6 +10,7 @@
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
+#include <linux/swap-prefetch.h>
#include <linux/init.h>
#include <linux/pagemap.h>
#include <linux/buffer_head.h>
@@ -95,7 +96,7 @@
return error;
}
-static int add_to_swap_cache(struct page *page, swp_entry_t entry)
+int add_to_swap_cache(struct page *page, swp_entry_t entry)
{
int error;
@@ -148,6 +149,9 @@
swp_entry_t entry;
int err;
+ /* Swap prefetching is delayed if we're swapping pages */
+ delay_swap_prefetch();
+
BUG_ON(!PageLocked(page));
for (;;) {
@@ -320,6 +324,9 @@
struct page *found_page, *new_page = NULL;
int err;
+ /* Swap prefetching is delayed if we're already reading from swap */
+ delay_swap_prefetch();
+
do {
/*
* First check the swap cache. Since this is normally
Index: linux-2.6.22-ck1/mm/vmscan.c
===================================================================
--- linux-2.6.22-ck1.orig/mm/vmscan.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/mm/vmscan.c 2007-07-10 14:55:23.000000000 +1000
@@ -16,6 +16,7 @@
#include <linux/slab.h>
#include <linux/kernel_stat.h>
#include <linux/swap.h>
+#include <linux/swap-prefetch.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/highmem.h>
@@ -36,6 +37,7 @@
#include <linux/rwsem.h>
#include <linux/delay.h>
#include <linux/kthread.h>
+#include <linux/timer.h>
#include <linux/freezer.h>
#include <asm/tlbflush.h>
@@ -63,7 +65,7 @@
* whole list at once. */
int swap_cluster_max;
- int swappiness;
+ int mapped;
int all_unreclaimable;
};
@@ -110,9 +112,10 @@
#endif
/*
- * From 0 .. 100. Higher means more swappy.
+ * From 0 .. 100. Lower means more swappy.
*/
-int vm_swappiness = 60;
+int vm_mapped __read_mostly = 66;
+int vm_hardmaplimit __read_mostly = 1;
long vm_total_pages; /* The total number of pages which the VM controls */
static LIST_HEAD(shrinker_list);
@@ -803,10 +806,14 @@
* The distress ratio is important - we don't want to start
* going oom.
*
- * A 100% value of vm_swappiness overrides this algorithm
- * altogether.
+ * This distress value is ignored if we apply a hardmaplimit except
+ * in extreme distress.
+ *
+ * A 0% value of vm_mapped overrides this algorithm altogether.
*/
- swap_tendency = mapped_ratio / 2 + distress + sc->swappiness;
+ swap_tendency = mapped_ratio * 100 / (sc->mapped + 1);
+ if (!vm_hardmaplimit || distress == 100)
+ swap_tendency += distress;
/*
* Now use this metric to decide whether to start moving mapped
@@ -955,6 +962,41 @@
}
/*
+ * Helper functions to adjust nice level of kswapd, based on the priority of
+ * the task (p) that called it. If it is already higher priority we do not
+ * demote its nice level since it is still working on behalf of a higher
+ * priority task. With kernel threads we leave it at nice 0.
+ *
+ * We don't ever run kswapd real time, so if a real time task calls kswapd we
+ * set it to highest SCHED_NORMAL priority.
+ */
+static int effective_sc_prio(struct task_struct *p)
+{
+ if (likely(p->mm)) {
+ if (rt_task(p))
+ return -20;
+ if (idleprio_task(p))
+ return 19;
+ return task_nice(p);
+ }
+ return 0;
+}
+
+static void set_kswapd_nice(struct task_struct *kswapd, struct task_struct *p,
+ int active)
+{
+ long nice = effective_sc_prio(p);
+
+ if (task_nice(kswapd) > nice || !active)
+ set_user_nice(kswapd, nice);
+}
+
+static int sc_priority(struct task_struct *p)
+{
+ return (DEF_PRIORITY + (DEF_PRIORITY * effective_sc_prio(p) / 40));
+}
+
+/*
* This is the direct reclaim path, for page-allocating processes. We only
* try to reclaim pages from zones which will satisfy the caller's allocation
* request.
@@ -1011,7 +1053,8 @@
* holds filesystem locks which prevent writeout this might not work, and the
* allocation attempt will fail.
*/
-unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask)
+unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask,
+ struct task_struct *p)
{
int priority;
int ret = 0;
@@ -1019,15 +1062,20 @@
unsigned long nr_reclaimed = 0;
struct reclaim_state *reclaim_state = current->reclaim_state;
unsigned long lru_pages = 0;
- int i;
+ int i, scan_priority = DEF_PRIORITY;
struct scan_control sc = {
.gfp_mask = gfp_mask,
.may_writepage = !laptop_mode,
.swap_cluster_max = SWAP_CLUSTER_MAX,
.may_swap = 1,
- .swappiness = vm_swappiness,
+ .mapped = vm_mapped,
};
+ if (p)
+ scan_priority = sc_priority(p);
+
+ delay_swap_prefetch();
+
count_vm_event(ALLOCSTALL);
for (i = 0; zones[i] != NULL; i++) {
@@ -1040,7 +1088,7 @@
+ zone_page_state(zone, NR_INACTIVE);
}
- for (priority = DEF_PRIORITY; priority >= 0; priority--) {
+ for (priority = scan_priority; priority >= 0; priority--) {
sc.nr_scanned = 0;
if (!priority)
disable_swap_token();
@@ -1070,7 +1118,7 @@
}
/* Take a nap, wait for some writeback to complete */
- if (sc.nr_scanned && priority < DEF_PRIORITY - 2)
+ if (sc.nr_scanned && priority < scan_priority - 2)
congestion_wait(WRITE, HZ/10);
}
/* top priority shrink_caches still had more to do? don't OOM, then */
@@ -1120,9 +1168,9 @@
*/
static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
{
- int all_zones_ok;
+ int all_zones_ok = 0;
int priority;
- int i;
+ int i, scan_priority;
unsigned long total_scanned;
unsigned long nr_reclaimed;
struct reclaim_state *reclaim_state = current->reclaim_state;
@@ -1130,7 +1178,7 @@
.gfp_mask = GFP_KERNEL,
.may_swap = 1,
.swap_cluster_max = SWAP_CLUSTER_MAX,
- .swappiness = vm_swappiness,
+ .mapped = vm_mapped,
};
/*
* temp_priority is used to remember the scanning priority at which
@@ -1138,6 +1186,8 @@
*/
int temp_priority[MAX_NR_ZONES];
+ scan_priority = sc_priority(pgdat->kswapd);
+
loop_again:
total_scanned = 0;
nr_reclaimed = 0;
@@ -1145,9 +1195,9 @@
count_vm_event(PAGEOUTRUN);
for (i = 0; i < pgdat->nr_zones; i++)
- temp_priority[i] = DEF_PRIORITY;
+ temp_priority[i] = scan_priority;
- for (priority = DEF_PRIORITY; priority >= 0; priority--) {
+ for (priority = scan_priority; priority >= 0; priority--) {
int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */
unsigned long lru_pages = 0;
@@ -1163,15 +1213,22 @@
*/
for (i = pgdat->nr_zones - 1; i >= 0; i--) {
struct zone *zone = pgdat->node_zones + i;
+ unsigned long watermark;
if (!populated_zone(zone))
continue;
- if (zone->all_unreclaimable && priority != DEF_PRIORITY)
+ if (zone->all_unreclaimable && priority != scan_priority)
continue;
- if (!zone_watermark_ok(zone, order, zone->pages_high,
- 0, 0)) {
+ /*
+ * The watermark is relaxed depending on the
+ * level of "priority" till it drops to
+ * pages_high.
+ */
+ watermark = zone->pages_high + (zone->pages_high *
+ priority / scan_priority);
+ if (!zone_watermark_ok(zone, order, watermark, 0, 0)) {
end_zone = i;
break;
}
@@ -1198,14 +1255,18 @@
for (i = 0; i <= end_zone; i++) {
struct zone *zone = pgdat->node_zones + i;
int nr_slab;
+ unsigned long watermark;
if (!populated_zone(zone))
continue;
- if (zone->all_unreclaimable && priority != DEF_PRIORITY)
+ if (zone->all_unreclaimable && priority != scan_priority)
continue;
- if (!zone_watermark_ok(zone, order, zone->pages_high,
+ watermark = zone->pages_high + (zone->pages_high *
+ priority / scan_priority);
+
+ if (!zone_watermark_ok(zone, order, watermark,
end_zone, 0))
all_zones_ok = 0;
temp_priority[i] = priority;
@@ -1238,7 +1299,7 @@
* OK, kswapd is getting into trouble. Take a nap, then take
* another pass across the zones.
*/
- if (total_scanned && priority < DEF_PRIORITY - 2)
+ if (total_scanned && priority < scan_priority - 2)
congestion_wait(WRITE, HZ/10);
/*
@@ -1272,6 +1333,8 @@
return nr_reclaimed;
}
+#define WT_EXPIRY (HZ * 5) /* Time to wakeup watermark_timer */
+
/*
* The background pageout daemon, started as a kernel thread
* from the init process.
@@ -1319,6 +1382,8 @@
for ( ; ; ) {
unsigned long new_order;
+ /* kswapd has been busy so delay watermark_timer */
+ mod_timer(&pgdat->watermark_timer, jiffies + WT_EXPIRY);
prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
new_order = pgdat->kswapd_max_order;
pgdat->kswapd_max_order = 0;
@@ -1332,6 +1397,7 @@
if (!freezing(current))
schedule();
+ set_user_nice(tsk, 0);
order = pgdat->kswapd_max_order;
}
finish_wait(&pgdat->kswapd_wait, &wait);
@@ -1349,9 +1415,10 @@
/*
* A zone is low on free memory, so wake its kswapd task to service it.
*/
-void wakeup_kswapd(struct zone *zone, int order)
+void wakeup_kswapd(struct zone *zone, int order, struct task_struct *p)
{
pg_data_t *pgdat;
+ int active;
if (!populated_zone(zone))
return;
@@ -1363,7 +1430,9 @@
pgdat->kswapd_max_order = order;
if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
return;
- if (!waitqueue_active(&pgdat->kswapd_wait))
+ active = waitqueue_active(&pgdat->kswapd_wait);
+ set_kswapd_nice(pgdat->kswapd, p, active);
+ if (!active)
return;
wake_up_interruptible(&pgdat->kswapd_wait);
}
@@ -1382,6 +1451,8 @@
struct zone *zone;
unsigned long nr_to_scan, ret = 0;
+ delay_swap_prefetch();
+
for_each_zone(zone) {
if (!populated_zone(zone))
@@ -1441,7 +1512,7 @@
.may_swap = 0,
.swap_cluster_max = nr_pages,
.may_writepage = 1,
- .swappiness = vm_swappiness,
+ .mapped = vm_mapped,
};
current->reclaim_state = &reclaim_state;
@@ -1476,7 +1547,7 @@
/* Force reclaiming mapped pages in the passes #3 and #4 */
if (pass > 2) {
sc.may_swap = 1;
- sc.swappiness = 100;
+ sc.mapped = 0;
}
for (prio = DEF_PRIORITY; prio >= 0; prio--) {
@@ -1540,20 +1611,57 @@
}
/*
+ * We wake up kswapd every WT_EXPIRY till free ram is above pages_lots
+ */
+static void watermark_wakeup(unsigned long data)
+{
+ pg_data_t *pgdat = (pg_data_t *)data;
+ struct timer_list *wt = &pgdat->watermark_timer;
+ int i;
+
+ if (!waitqueue_active(&pgdat->kswapd_wait) || above_background_load())
+ goto out;
+ for (i = pgdat->nr_zones - 1; i >= 0; i--) {
+ struct zone *z = pgdat->node_zones + i;
+
+ if (!populated_zone(z) || is_highmem(z)) {
+ /* We are better off leaving highmem full */
+ continue;
+ }
+ if (!zone_watermark_ok(z, 0, z->pages_lots, 0, 0)) {
+ wake_up_interruptible(&pgdat->kswapd_wait);
+ goto out;
+ }
+ }
+out:
+ mod_timer(wt, jiffies + WT_EXPIRY);
+ return;
+}
+
+/*
* This kswapd start function will be called by init and node-hot-add.
* On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
*/
int kswapd_run(int nid)
{
pg_data_t *pgdat = NODE_DATA(nid);
+ struct timer_list *wt;
int ret = 0;
if (pgdat->kswapd)
return 0;
+ wt = &pgdat->watermark_timer;
+ init_timer(wt);
+ wt->data = (unsigned long)pgdat;
+ wt->function = watermark_wakeup;
+ wt->expires = jiffies + WT_EXPIRY;
+ add_timer(wt);
+
pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
if (IS_ERR(pgdat->kswapd)) {
/* failure at boot is fatal */
+ del_timer(wt);
BUG_ON(system_state == SYSTEM_BOOTING);
printk("Failed to start kswapd on node %d\n",nid);
ret = -1;
@@ -1624,7 +1732,7 @@
.swap_cluster_max = max_t(unsigned long, nr_pages,
SWAP_CLUSTER_MAX),
.gfp_mask = gfp_mask,
- .swappiness = vm_swappiness,
+ .mapped = vm_mapped,
};
unsigned long slab_reclaimable;
Index: linux-2.6.22-ck1/include/linux/mm_inline.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/mm_inline.h 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/mm_inline.h 2007-07-10 14:55:22.000000000 +1000
@@ -13,6 +13,13 @@
}
static inline void
+add_page_to_inactive_list_tail(struct zone *zone, struct page *page)
+{
+ list_add_tail(&page->lru, &zone->inactive_list);
+ __inc_zone_state(zone, NR_INACTIVE);
+}
+
+static inline void
del_page_from_active_list(struct zone *zone, struct page *page)
{
list_del(&page->lru);
Index: linux-2.6.22-ck1/include/linux/swap-prefetch.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux-2.6.22-ck1/include/linux/swap-prefetch.h 2007-07-10 14:55:22.000000000 +1000
@@ -0,0 +1,53 @@
+#ifndef SWAP_PREFETCH_H_INCLUDED
+#define SWAP_PREFETCH_H_INCLUDED
+
+#ifdef CONFIG_SWAP_PREFETCH
+/* mm/swap_prefetch.c */
+extern int swap_prefetch;
+extern int swap_prefetch_delay;
+extern int swap_prefetch_sleep;
+
+struct swapped_entry {
+ swp_entry_t swp_entry; /* The actual swap entry */
+ struct list_head swapped_list; /* Linked list of entries */
+#if MAX_NUMNODES > 1
+ int node; /* Node id */
+#endif
+} __attribute__((packed));
+
+static inline void store_swap_entry_node(struct swapped_entry *entry,
+ struct page *page)
+{
+#if MAX_NUMNODES > 1
+ entry->node = page_to_nid(page);
+#endif
+}
+
+static inline int get_swap_entry_node(struct swapped_entry *entry)
+{
+#if MAX_NUMNODES > 1
+ return entry->node;
+#else
+ return 0;
+#endif
+}
+
+extern void add_to_swapped_list(struct page *page);
+extern void delay_swap_prefetch(void);
+extern void prepare_swap_prefetch(void);
+
+#else /* CONFIG_SWAP_PREFETCH */
+static inline void add_to_swapped_list(struct page *__unused)
+{
+}
+
+static inline void prepare_swap_prefetch(void)
+{
+}
+
+static inline void delay_swap_prefetch(void)
+{
+}
+#endif /* CONFIG_SWAP_PREFETCH */
+
+#endif /* SWAP_PREFETCH_H_INCLUDED */
Index: linux-2.6.22-ck1/mm/page_io.c
===================================================================
--- linux-2.6.22-ck1.orig/mm/page_io.c 2007-07-10 14:55:00.000000000 +1000
+++ linux-2.6.22-ck1/mm/page_io.c 2007-07-10 14:55:22.000000000 +1000
@@ -17,6 +17,7 @@
#include <linux/bio.h>
#include <linux/swapops.h>
#include <linux/writeback.h>
+#include <linux/swap-prefetch.h>
#include <asm/pgtable.h>
static struct bio *get_swap_bio(gfp_t gfp_flags, pgoff_t index,
@@ -118,6 +119,7 @@
ret = -ENOMEM;
goto out;
}
+ add_to_swapped_list(page);
if (wbc->sync_mode == WB_SYNC_ALL)
rw |= (1 << BIO_RW_SYNC);
count_vm_event(PSWPOUT);
Index: linux-2.6.22-ck1/include/linux/sysctl.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/sysctl.h 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/sysctl.h 2007-07-10 14:55:22.000000000 +1000
@@ -190,7 +190,7 @@
VM_OVERCOMMIT_RATIO=16, /* percent of RAM to allow overcommit in */
VM_PAGEBUF=17, /* struct: Control pagebuf parameters */
VM_HUGETLB_PAGES=18, /* int: Number of available Huge Pages */
- VM_SWAPPINESS=19, /* Tendency to steal mapped memory */
+ VM_UNUSED19=19, /* was: Tendency to steal mapped memory */
VM_LOWMEM_RESERVE_RATIO=20,/* reservation ratio for lower memory zones */
VM_MIN_FREE_KBYTES=21, /* Minimum free kilobytes to maintain */
VM_MAX_MAP_COUNT=22, /* int: Maximum number of mmaps/address-space */
Index: linux-2.6.22-ck1/include/linux/mmzone.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/mmzone.h 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/mmzone.h 2007-07-10 14:55:23.000000000 +1000
@@ -13,6 +13,7 @@
#include <linux/init.h>
#include <linux/seqlock.h>
#include <linux/nodemask.h>
+#include <linux/timer.h>
#include <asm/atomic.h>
#include <asm/page.h>
@@ -181,7 +182,7 @@
struct zone {
/* Fields commonly accessed by the page allocator */
- unsigned long pages_min, pages_low, pages_high;
+ unsigned long pages_min, pages_low, pages_high, pages_lots;
/*
* We don't know if the memory that we're going to allocate will be freeable
* or/and it will be released eventually, so to avoid totally wasting several
@@ -452,6 +453,7 @@
wait_queue_head_t kswapd_wait;
struct task_struct *kswapd;
int kswapd_max_order;
+ struct timer_list watermark_timer;
} pg_data_t;
#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
@@ -468,7 +470,7 @@
void get_zone_counts(unsigned long *active, unsigned long *inactive,
unsigned long *free);
void build_all_zonelists(void);
-void wakeup_kswapd(struct zone *zone, int order);
+void wakeup_kswapd(struct zone *zone, int order, struct task_struct *p);
int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
int classzone_idx, int alloc_flags);
enum memmap_context {
Index: linux-2.6.22-ck1/mm/page_alloc.c
===================================================================
--- linux-2.6.22-ck1.orig/mm/page_alloc.c 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/mm/page_alloc.c 2007-07-10 14:55:22.000000000 +1000
@@ -1250,7 +1250,7 @@
goto nopage;
for (z = zonelist->zones; *z; z++)
- wakeup_kswapd(*z, order);
+ wakeup_kswapd(*z, order, p);
/*
* OK, we're below the kswapd watermark and have kicked background
@@ -1314,7 +1314,7 @@
reclaim_state.reclaimed_slab = 0;
p->reclaim_state = &reclaim_state;
- did_some_progress = try_to_free_pages(zonelist->zones, gfp_mask);
+ did_some_progress = try_to_free_pages(zonelist->zones, gfp_mask, p);
p->reclaim_state = NULL;
p->flags &= ~PF_MEMALLOC;
@@ -1570,6 +1570,7 @@
" min:%lukB"
" low:%lukB"
" high:%lukB"
+ " lots:%lukB"
" active:%lukB"
" inactive:%lukB"
" present:%lukB"
@@ -1581,6 +1582,7 @@
K(zone->pages_min),
K(zone->pages_low),
K(zone->pages_high),
+ K(zone->pages_lots),
K(zone_page_state(zone, NR_ACTIVE)),
K(zone_page_state(zone, NR_INACTIVE)),
K(zone->present_pages),
@@ -3142,6 +3144,7 @@
zone->pages_low = zone->pages_min + (tmp >> 2);
zone->pages_high = zone->pages_min + (tmp >> 1);
+ zone->pages_lots = zone->pages_min + tmp;
spin_unlock_irqrestore(&zone->lru_lock, flags);
}
Index: linux-2.6.22-ck1/fs/buffer.c
===================================================================
--- linux-2.6.22-ck1.orig/fs/buffer.c 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/fs/buffer.c 2007-07-10 14:55:22.000000000 +1000
@@ -356,7 +356,7 @@
for_each_online_pgdat(pgdat) {
zones = pgdat->node_zonelists[gfp_zone(GFP_NOFS)].zones;
if (*zones)
- try_to_free_pages(zones, GFP_NOFS);
+ try_to_free_pages(zones, GFP_NOFS, NULL);
}
}
Index: linux-2.6.22-ck1/mm/filemap.c
===================================================================
--- linux-2.6.22-ck1.orig/mm/filemap.c 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/mm/filemap.c 2007-07-10 14:55:23.000000000 +1000
@@ -466,6 +466,16 @@
return ret;
}
+int add_to_page_cache_lru_tail(struct page *page,
+ struct address_space *mapping, pgoff_t offset, gfp_t gfp_mask)
+{
+ int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
+
+ if (ret == 0)
+ lru_cache_add_tail(page);
+ return ret;
+}
+
#ifdef CONFIG_NUMA
struct page *__page_cache_alloc(gfp_t gfp)
{
@@ -839,6 +849,34 @@
ra->ra_pages /= 4;
}
+/*
+ * Sysctl which determines whether we should read from large files to the
+ * tail of the inactive lru list.
+ */
+int vm_tail_largefiles __read_mostly = 1;
+
+static inline int nr_mapped(void)
+{
+ return global_page_state(NR_FILE_MAPPED) +
+ global_page_state(NR_ANON_PAGES);
+}
+
+/*
+ * This examines how large in pages a file size is and returns 1 if it is
+ * more than half the unmapped ram. Avoid doing read_page_state which is
+ * expensive unless we already know it is likely to be large enough.
+ */
+static int large_isize(unsigned long nr_pages)
+{
+ if (nr_pages * 6 > vm_total_pages) {
+ unsigned long unmapped_ram = vm_total_pages - nr_mapped();
+
+ if (nr_pages * 2 > unmapped_ram)
+ return 1;
+ }
+ return 0;
+}
+
/**
* do_generic_mapping_read - generic file read routine
* @mapping: address_space to be read
@@ -1051,8 +1089,19 @@
goto out;
}
}
- error = add_to_page_cache_lru(cached_page, mapping,
- index, GFP_KERNEL);
+
+ /*
+ * If we know the file is large we add the pages read to the
+ * end of the lru as we're unlikely to be able to cache the
+ * whole file in ram so make those pages the first to be
+ * dropped if not referenced soon.
+ */
+ if (vm_tail_largefiles && large_isize(end_index))
+ error = add_to_page_cache_lru_tail(cached_page,
+ mapping, index, GFP_KERNEL);
+ else
+ error = add_to_page_cache_lru(cached_page, mapping,
+ index, GFP_KERNEL);
if (error) {
if (error == -EEXIST)
goto find_page;
Index: linux-2.6.22-ck1/Documentation/filesystems/proc.txt
===================================================================
--- linux-2.6.22-ck1.orig/Documentation/filesystems/proc.txt 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/Documentation/filesystems/proc.txt 2007-07-10 14:55:23.000000000 +1000
@@ -1333,6 +1333,14 @@
As this is a non-destructive operation and dirty objects are not freeable, the
user should run `sync' first.
+tail_largefiles
+---------------
+
+When enabled reads from large files to the tail end of the inactive lru list.
+This means that any cache from reading large files is dropped very quickly,
+preventing loss of mapped ram and useful pagecache when large files are read.
+This does, however, make caching less effective when working with large files.
+
2.5 /proc/sys/dev - Device specific parameters
----------------------------------------------
Index: linux-2.6.22-ck1/arch/i386/Kconfig
===================================================================
--- linux-2.6.22-ck1.orig/arch/i386/Kconfig 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/arch/i386/Kconfig 2007-07-10 14:55:23.000000000 +1000
@@ -550,7 +550,7 @@
choice
depends on EXPERIMENTAL
- prompt "Memory split" if EMBEDDED
+ prompt "Memory split"
default VMSPLIT_3G
help
Select the desired split between kernel and user memory.
@@ -569,17 +569,17 @@
option alone!
config VMSPLIT_3G
- bool "3G/1G user/kernel split"
+ bool "Default 896MB lowmem (3G/1G user/kernel split)"
config VMSPLIT_3G_OPT
depends on !HIGHMEM
- bool "3G/1G user/kernel split (for full 1G low memory)"
+ bool "1GB lowmem (3G/1G user/kernel split)"
config VMSPLIT_2G
- bool "2G/2G user/kernel split"
+ bool "2GB lowmem (2G/2G user/kernel split)"
config VMSPLIT_2G_OPT
depends on !HIGHMEM
- bool "2G/2G user/kernel split (for full 2G low memory)"
+ bool "2GB lowmem (2G/2G user/kernel split)"
config VMSPLIT_1G
- bool "1G/3G user/kernel split"
+ bool "3GB lowmem (1G/3G user/kernel split)"
endchoice
config PAGE_OFFSET
Index: linux-2.6.22-ck1/kernel/Kconfig.hz
===================================================================
--- linux-2.6.22-ck1.orig/kernel/Kconfig.hz 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/kernel/Kconfig.hz 2007-07-10 14:55:24.000000000 +1000
@@ -4,7 +4,7 @@
choice
prompt "Timer frequency"
- default HZ_250
+ default HZ_1000
help
Allows the configuration of the timer frequency. It is customary
to have the timer interrupt run at 1000 Hz but 100 Hz may be more
@@ -13,8 +13,7 @@
contention and cacheline bounces as a result of timer interrupts.
Note that the timer interrupt occurs on each processor in an SMP
environment leading to NR_CPUS * HZ number of timer interrupts
- per second.
-
+ per second.Laptops may also show improved battery life.
config HZ_100
bool "100 HZ"
@@ -23,13 +22,14 @@
with lots of processors that may show reduced performance if
too many timer interrupts are occurring.
- config HZ_250
+ config HZ_250_NODEFAULT
bool "250 HZ"
help
- 250 Hz is a good compromise choice allowing server performance
- while also showing good interactive responsiveness even
- on SMP and NUMA systems. If you are going to be using NTSC video
- or multimedia, selected 300Hz instead.
+ 250 HZ is a lousy compromise choice allowing server interactivity
+ while also showing desktop throughput and no extra power saving on
+ laptops. Good for when you can't make up your mind.
+
+ Recommend 100 or 1000 instead.
config HZ_300
bool "300 HZ"
@@ -45,12 +45,76 @@
1000 Hz is the preferred choice for desktop systems and other
systems requiring fast interactive responses to events.
+ config HZ_1500
+ bool "1500 HZ"
+ help
+ 1500 Hz is an insane value to use to run broken software that is Hz
+ limited.
+
+ Being over 1000, driver breakage is likely.
+
+ config HZ_2000
+ bool "2000 HZ"
+ help
+ 2000 Hz is an insane value to use to run broken software that is Hz
+ limited.
+
+ Being over 1000, driver breakage is likely.
+
+ config HZ_3000
+ bool "3000 HZ"
+ help
+ 3000 Hz is an insane value to use to run broken software that is Hz
+ limited.
+
+ Being over 1000, driver breakage is likely.
+
+ config HZ_4000
+ bool "4000 HZ"
+ help
+ 4000 Hz is an insane value to use to run broken software that is Hz
+ limited.
+
+ Being over 1000, driver breakage is likely.
+
+ config HZ_5000
+ bool "5000 HZ"
+ help
+ 5000 Hz is an obscene value to use to run broken software that is Hz
+ limited.
+
+ Being over 1000, driver breakage is likely.
+
+ config HZ_7500
+ bool "7500 HZ"
+ help
+ 7500 Hz is an obscene value to use to run broken software that is Hz
+ limited.
+
+ Being over 1000, driver breakage is likely.
+
+ config HZ_10000
+ bool "10000 HZ"
+ help
+ 10000 Hz is an obscene value to use to run broken software that is Hz
+ limited.
+
+ Being over 1000, driver breakage is likely.
+
+
endchoice
config HZ
int
default 100 if HZ_100
- default 250 if HZ_250
+ default 250 if HZ_250_NODEFAULT
default 300 if HZ_300
default 1000 if HZ_1000
+ default 1500 if HZ_1500
+ default 2000 if HZ_2000
+ default 3000 if HZ_3000
+ default 4000 if HZ_4000
+ default 5000 if HZ_5000
+ default 7500 if HZ_7500
+ default 10000 if HZ_10000
Index: linux-2.6.22-ck1/arch/i386/defconfig
===================================================================
--- linux-2.6.22-ck1.orig/arch/i386/defconfig 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/arch/i386/defconfig 2007-07-10 14:55:23.000000000 +1000
@@ -226,10 +226,10 @@
# CONFIG_IRQBALANCE is not set
CONFIG_SECCOMP=y
# CONFIG_HZ_100 is not set
-CONFIG_HZ_250=y
+# CONFIG_HZ_250 is not set
# CONFIG_HZ_300 is not set
-# CONFIG_HZ_1000 is not set
-CONFIG_HZ=250
+CONFIG_HZ_1000=y
+CONFIG_HZ=1000
# CONFIG_KEXEC is not set
# CONFIG_CRASH_DUMP is not set
CONFIG_PHYSICAL_START=0x100000
Index: linux-2.6.22-ck1/arch/x86_64/defconfig
===================================================================
--- linux-2.6.22-ck1.orig/arch/x86_64/defconfig 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/arch/x86_64/defconfig 2007-07-10 14:55:23.000000000 +1000
@@ -185,10 +185,10 @@
CONFIG_SECCOMP=y
# CONFIG_CC_STACKPROTECTOR is not set
# CONFIG_HZ_100 is not set
-CONFIG_HZ_250=y
+# CONFIG_HZ_250 is not set
# CONFIG_HZ_300 is not set
-# CONFIG_HZ_1000 is not set
-CONFIG_HZ=250
+CONFIG_HZ_1000=y
+CONFIG_HZ=1000
CONFIG_K8_NB=y
CONFIG_GENERIC_HARDIRQS=y
CONFIG_GENERIC_IRQ_PROBE=y
Index: linux-2.6.22-ck1/include/linux/jiffies.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/jiffies.h 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/jiffies.h 2007-07-10 14:55:24.000000000 +1000
@@ -29,6 +29,12 @@
# define SHIFT_HZ 9
#elif HZ >= 768 && HZ < 1536
# define SHIFT_HZ 10
+#elif HZ >= 1536 && HZ < 3072
+# define SHIFT_HZ 11
+#elif HZ >= 3072 && HZ < 6144
+# define SHIFT_HZ 12
+#elif HZ >= 6144 && HZ < 12288
+# define SHIFT_HZ 13
#else
# error You lose.
#endif
Index: linux-2.6.22-ck1/include/net/inet_timewait_sock.h
===================================================================
--- linux-2.6.22-ck1.orig/include/net/inet_timewait_sock.h 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/include/net/inet_timewait_sock.h 2007-07-10 14:55:24.000000000 +1000
@@ -38,8 +38,8 @@
* If time > 4sec, it is "slow" path, no recycling is required,
* so that we select tick to get range about 4 seconds.
*/
-#if HZ <= 16 || HZ > 4096
-# error Unsupported: HZ <= 16 or HZ > 4096
+#if HZ <= 16 || HZ > 16384
+# error Unsupported: HZ <= 16 or HZ > 16384
#elif HZ <= 32
# define INET_TWDR_RECYCLE_TICK (5 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
#elif HZ <= 64
@@ -54,8 +54,12 @@
# define INET_TWDR_RECYCLE_TICK (10 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
#elif HZ <= 2048
# define INET_TWDR_RECYCLE_TICK (11 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
-#else
+#elif HZ <= 4096
# define INET_TWDR_RECYCLE_TICK (12 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
+#elif HZ <= 8192
+# define INET_TWDR_RECYCLE_TICK (13 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
+#else
+# define INET_TWDR_RECYCLE_TICK (14 + 2 - INET_TWDR_RECYCLE_SLOTS_LOG)
#endif
/* TIME_WAIT reaping mechanism. */
Index: linux-2.6.22-ck1/init/calibrate.c
===================================================================
--- linux-2.6.22-ck1.orig/init/calibrate.c 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/init/calibrate.c 2007-07-10 14:55:24.000000000 +1000
@@ -122,12 +122,12 @@
printk("Calibrating delay loop (skipped)... "
"%lu.%02lu BogoMIPS preset\n",
loops_per_jiffy/(500000/HZ),
- (loops_per_jiffy/(5000/HZ)) % 100);
+ (loops_per_jiffy * 10/(50000/HZ)) % 100);
} else if ((loops_per_jiffy = calibrate_delay_direct()) != 0) {
printk("Calibrating delay using timer specific routine.. ");
printk("%lu.%02lu BogoMIPS (lpj=%lu)\n",
loops_per_jiffy/(500000/HZ),
- (loops_per_jiffy/(5000/HZ)) % 100,
+ (loops_per_jiffy * 10/(50000/HZ)) % 100,
loops_per_jiffy);
} else {
loops_per_jiffy = (1<<12);
@@ -166,7 +166,7 @@
/* Round the value and print it */
printk("%lu.%02lu BogoMIPS (lpj=%lu)\n",
loops_per_jiffy/(500000/HZ),
- (loops_per_jiffy/(5000/HZ)) % 100,
+ (loops_per_jiffy * 10/(50000/HZ)) % 100,
loops_per_jiffy);
}
Index: linux-2.6.22-ck1/arch/i386/kernel/cpu/proc.c
===================================================================
--- linux-2.6.22-ck1.orig/arch/i386/kernel/cpu/proc.c 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/arch/i386/kernel/cpu/proc.c 2007-07-10 14:55:24.000000000 +1000
@@ -157,7 +157,7 @@
seq_printf(m, "\nbogomips\t: %lu.%02lu\n",
c->loops_per_jiffy/(500000/HZ),
- (c->loops_per_jiffy/(5000/HZ)) % 100);
+ (c->loops_per_jiffy * 10/(50000/HZ)) % 100);
seq_printf(m, "clflush size\t: %u\n\n", c->x86_clflush_size);
return 0;
Index: linux-2.6.22-ck1/arch/i386/kernel/smpboot.c
===================================================================
--- linux-2.6.22-ck1.orig/arch/i386/kernel/smpboot.c 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/arch/i386/kernel/smpboot.c 2007-07-10 14:55:24.000000000 +1000
@@ -1094,7 +1094,7 @@
"Total of %d processors activated (%lu.%02lu BogoMIPS).\n",
cpucount+1,
bogosum/(500000/HZ),
- (bogosum/(5000/HZ))%100);
+ (bogosum * 10/(50000/HZ))%100);
Dprintk("Before bogocount - setting activated=1.\n");
Index: linux-2.6.22-ck1/include/linux/nfsd/stats.h
===================================================================
--- linux-2.6.22-ck1.orig/include/linux/nfsd/stats.h 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/include/linux/nfsd/stats.h 2007-07-10 14:55:24.000000000 +1000
@@ -35,8 +35,8 @@
};
-/* thread usage wraps very million seconds (approx one fortnight) */
-#define NFSD_USAGE_WRAP (HZ*1000000)
+/* thread usage wraps every one hundred thousand seconds (approx one day) */
+#define NFSD_USAGE_WRAP (HZ*100000)
#ifdef __KERNEL__
Index: linux-2.6.22-ck1/arch/x86_64/kernel/setup.c
===================================================================
--- linux-2.6.22-ck1.orig/arch/x86_64/kernel/setup.c 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/arch/x86_64/kernel/setup.c 2007-07-10 14:55:24.000000000 +1000
@@ -1047,7 +1047,7 @@
seq_printf(m, "\nbogomips\t: %lu.%02lu\n",
c->loops_per_jiffy/(500000/HZ),
- (c->loops_per_jiffy/(5000/HZ)) % 100);
+ (c->loops_per_jiffy * 10/(50000/HZ)) % 100);
if (c->x86_tlbsize > 0)
seq_printf(m, "TLB size\t: %d 4K pages\n", c->x86_tlbsize);
Index: linux-2.6.22-ck1/Makefile
===================================================================
--- linux-2.6.22-ck1.orig/Makefile 2007-07-10 14:54:59.000000000 +1000
+++ linux-2.6.22-ck1/Makefile 2007-07-10 14:55:24.000000000 +1000
@@ -1,8 +1,9 @@
VERSION = 2
PATCHLEVEL = 6
SUBLEVEL = 22
-EXTRAVERSION =
-NAME = Holy Dancing Manatees, Batman!
+EXTRAVERSION = -ck1
+NAME = So long, and thanks for all the fish
+JANAME = さようなら、いままで魚をありがとう
# *DOCUMENTATION*
# To see a list of typical targets execute "make help"
Reference in a new issue Copy permalink