android_kernel_samsung_sm8750/kernel/sched/walt/walt.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
 * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved.
 */

#ifndef _WALT_H
#define _WALT_H

#include <linux/pm_qos.h>
#include <linux/sched/cputime.h>
#include "../../../kernel/sched/sched.h"
#include "../../../fs/proc/internal.h"
#include <linux/sched/walt.h>
#include <linux/jump_label.h>

#include <linux/cgroup.h>

#include <uapi/linux/sched/types.h>
#include <linux/cpuidle.h>
#include <linux/sched/clock.h>
#include <trace/hooks/cgroup.h>

#define MSEC_TO_NSEC (1000 * 1000)

#ifdef CONFIG_HZ_300
/*
 * Tick interval becomes to 3333333 due to
 * rounding error when HZ=300.
 */
#define DEFAULT_SCHED_RAVG_WINDOW (3333333 * 5)
#else
/* Min window size (in ns) = 16ms */
#define DEFAULT_SCHED_RAVG_WINDOW 16000000
#endif

/* Max window size (in ns) = 1s */
#define MAX_SCHED_RAVG_WINDOW 1000000000

#define SCHED_RAVG_8MS_WINDOW 8000000
#define SCHED_RAVG_12MS_WINDOW 12000000
#define SCHED_RAVG_16MS_WINDOW 16000000
#define SCHED_RAVG_20MS_WINDOW 20000000
#define SCHED_RAVG_32MS_WINDOW 32000000

#define NR_WINDOWS_PER_SEC (NSEC_PER_SEC / DEFAULT_SCHED_RAVG_WINDOW)

/* MAX_MARGIN_LEVELS should be one less than MAX_CLUSTERS */
#define MAX_MARGIN_LEVELS (MAX_CLUSTERS - 1)

extern bool walt_disabled;
extern bool waltgov_disabled;
extern bool trailblazer_state;

enum task_event {
	PUT_PREV_TASK	= 0,
	PICK_NEXT_TASK	= 1,
	TASK_WAKE	= 2,
	TASK_MIGRATE	= 3,
	TASK_UPDATE	= 4,
	IRQ_UPDATE	= 5,
};

/* Note: this need to be in sync with migrate_type_names array */
enum migrate_types {
	GROUP_TO_RQ,
	RQ_TO_GROUP,
};

enum pipeline_types {
	MANUAL_PIPELINE,
	AUTO_PIPELINE,
	MAX_PIPELINE_TYPES,
};

enum freq_caps {
	PARTIAL_HALT_CAP,
	SMART_FREQ,
	HIGH_PERF_CAP,
	MAX_FREQ_CAP,
};

/*soc bit flags interface*/
#define	SOC_ENABLE_CONSERVATIVE_BOOST_TOPAPP_BIT	BIT(0)
#define	SOC_ENABLE_CONSERVATIVE_BOOST_FG_BIT		BIT(1)
#define	SOC_ENABLE_UCLAMP_BOOSTED_BIT			BIT(2)
#define	SOC_ENABLE_PER_TASK_BOOST_ON_MID_BIT		BIT(3)
#define	SOC_ENABLE_SILVER_RT_SPREAD_BIT			BIT(4)
#define	SOC_ENABLE_BOOST_TO_NEXT_CLUSTER_BIT		BIT(5)
#define	SOC_ENABLE_SW_CYCLE_COUNTER_BIT			BIT(6)
#define SOC_ENABLE_COLOCATION_PLACEMENT_BOOST_BIT	BIT(7)
#define SOC_ENABLE_FT_BOOST_TO_ALL			BIT(8)
#define SOC_ENABLE_EXPERIMENT3						BIT(9)
#define SOC_ENABLE_PIPELINE_SWAPPING_BIT		BIT(10)
#define SOC_ENABLE_THERMAL_HALT_LOW_FREQ_BIT		BIT(11)

extern int soc_sched_lib_name_capacity;

/* WALT feature */
#define WALT_FEAT_TRAILBLAZER_BIT	BIT_ULL(0)
#define WALT_FEAT_UCLAMP_FREQ_BIT	BIT_ULL(1)

extern unsigned int trailblazer_floor_freq[MAX_CLUSTERS];

/*wts->flags bits*/
#define	WALT_INIT_BIT			BIT(0)
#define WALT_TRAILBLAZER_BIT		BIT(1)
#define WALT_IDLE_TASK_BIT		BIT(2)
#define WALT_LRB_PIPELINE_BIT		BIT(3)

#define WALT_LOW_LATENCY_PROCFS_BIT	BIT(0)
#define WALT_LOW_LATENCY_BINDER_BIT	BIT(1)
#define WALT_LOW_LATENCY_PIPELINE_BIT	BIT(2)
#define WALT_LOW_LATENCY_HEAVY_BIT	BIT(3)

#define WALT_LOW_LATENCY_BIT_MASK	(WALT_LOW_LATENCY_PIPELINE_BIT|WALT_LOW_LATENCY_HEAVY_BIT)

struct walt_cpu_load {
	unsigned long	nl;
	unsigned long	pl;
	bool		rtgb_active;
	u64		ws;
	bool		ed_active;
	bool		trailblazer_state;
};

#define DECLARE_BITMAP_ARRAY(name, nr, bits) \
	unsigned long name[nr][BITS_TO_LONGS(bits)]

struct walt_sched_stats {
	int		nr_big_tasks;
	int		nr_trailblazer_tasks;
	u64		cumulative_runnable_avg_scaled;
	u64		pred_demands_sum_scaled;
	unsigned int	nr_rtg_high_prio_tasks;
};

#define NUM_TRACKED_WINDOWS 2
#define NUM_LOAD_INDICES 1000

struct group_cpu_time {
	u64			curr_runnable_sum;
	u64			prev_runnable_sum;
	u64			nt_curr_runnable_sum;
	u64			nt_prev_runnable_sum;
};

struct load_subtractions {
	u64			window_start;
	u64			subs;
	u64			new_subs;
};

/* ========================= SMART FREQ config =====================*/
enum smart_freq_legacy_reason {
	NO_REASON_SMART_FREQ,
	BOOST_SMART_FREQ,
	SUSTAINED_HIGH_UTIL_SMART_FREQ,
	BIG_TASKCNT_SMART_FREQ,
	TRAILBLAZER_SMART_FREQ,
	SBT_SMART_FREQ,
	PIPELINE_60FPS_OR_LESSER_SMART_FREQ,
	PIPELINE_90FPS_SMART_FREQ,
	PIPELINE_120FPS_OR_GREATER_SMART_FREQ,
	THERMAL_ROTATION_SMART_FREQ,
	LEGACY_SMART_FREQ,
};

enum smart_freq_ipc_reason {
	IPC_A,
	IPC_B,
	IPC_C,
	IPC_D,
	IPC_E,
	SMART_FMAX_IPC_MAX,
};
#define IPC_PARTICIPATION	(BIT(IPC_A) | BIT(IPC_B) | BIT(IPC_C) | BIT(IPC_D) | BIT(IPC_E))

DECLARE_PER_CPU(unsigned int, ipc_level);
DECLARE_PER_CPU(unsigned long, ipc_cnt);
DECLARE_PER_CPU(unsigned long, intr_cnt);
DECLARE_PER_CPU(unsigned long, cycle_cnt);
DECLARE_PER_CPU(u64, last_ipc_update);
DECLARE_PER_CPU(u64, ipc_deactivate_ns);
DECLARE_PER_CPU(bool, tickless_mode);

struct smart_freq_legacy_reason_status {
	u64 deactivate_ns;
};

struct smart_freq_legacy_reason_config {
	unsigned long freq_allowed;
	u64 hyst_ns;
};

struct smart_freq_ipc_reason_config {
	unsigned long ipc;
	unsigned long freq_allowed;
	u64 hyst_ns;
};

struct smart_freq_cluster_info {
	u32 smart_freq_participation_mask;
	unsigned int cluster_active_reason;
	unsigned int cluster_ipc_level;
	unsigned long min_cycles;
	u32 smart_freq_ipc_participation_mask;
	struct smart_freq_legacy_reason_config legacy_reason_config[LEGACY_SMART_FREQ];
	struct smart_freq_legacy_reason_status legacy_reason_status[LEGACY_SMART_FREQ];
	struct smart_freq_ipc_reason_config ipc_reason_config[SMART_FMAX_IPC_MAX];
};

extern bool smart_freq_init_done;
extern unsigned int big_task_cnt;
extern struct smart_freq_cluster_info default_freq_config[MAX_CLUSTERS];
/*=========================================================================*/
struct walt_sched_cluster {
	raw_spinlock_t		load_lock;
	struct list_head	list;
	struct cpumask		cpus;
	int			id;
	/*
	 * max_possible_freq = maximum supported by hardware
	 * max_freq = max freq as per cpufreq limits
	 */
	unsigned int		cur_freq;
	unsigned int		max_possible_freq;
	unsigned int		max_freq;
	unsigned int		walt_internal_freq_limit;
	u64			aggr_grp_load;
	unsigned long		util_to_cost[1024];
	u64			found_ts;
	struct smart_freq_cluster_info *smart_freq_info;
};

struct walt_rq {
	struct task_struct	*push_task;
	struct walt_sched_cluster *cluster;
	struct cpumask		freq_domain_cpumask;
	struct walt_sched_stats walt_stats;

	u64			window_start;
	u32			prev_window_size;
	unsigned long		walt_flags;

	u64			avg_irqload;
	u64			last_irq_window;
	u64			prev_irq_time;
	struct task_struct	*ed_task;
	u64			task_exec_scale;
	u64			old_busy_time;
	u64			old_estimated_time;
	u64			curr_runnable_sum;
	u64			prev_runnable_sum;
	u64			nt_curr_runnable_sum;
	u64			nt_prev_runnable_sum;
	struct group_cpu_time	grp_time;
	struct load_subtractions load_subs[NUM_TRACKED_WINDOWS];
	DECLARE_BITMAP_ARRAY(top_tasks_bitmap,
			NUM_TRACKED_WINDOWS, NUM_LOAD_INDICES);
	u8			*top_tasks[NUM_TRACKED_WINDOWS];
	u8			curr_table;
	int			prev_top;
	int			curr_top;
	bool			notif_pending;
	bool			high_irqload;
	u64			last_cc_update;
	u64			cycles;
	u64			util;
	/* MVP */
	struct list_head	mvp_tasks;
	int                     num_mvp_tasks;
	u64			mvp_arrival_time; /* ts when 1st mvp task selected on this cpu */
	u64			mvp_throttle_time; /* ts when mvp were throttled */
	bool			skip_mvp;

	u64			latest_clock;
	u32			enqueue_counter;
	u64			lrb_pipeline_start_time; /* lrb = long_running_boost */

	/* UCLAMP tracking */
	unsigned long		uclamp_limit[UCLAMP_CNT];
};

DECLARE_PER_CPU(struct walt_rq, walt_rq);

extern struct completion walt_get_cycle_counts_cb_completion;
extern bool use_cycle_counter;
extern struct walt_sched_cluster *sched_cluster[WALT_NR_CPUS];
extern cpumask_t part_haltable_cpus;
extern cpumask_t cpus_paused_by_us;
extern cpumask_t cpus_part_paused_by_us;
/*END SCHED.H PORT*/

extern u64 (*walt_get_cycle_counts_cb)(int cpu, u64 wc);
extern int walt_cpufreq_cycle_cntr_driver_register(void);
extern int walt_gclk_cycle_counter_driver_register(void);

extern int num_sched_clusters;
extern unsigned int sched_capacity_margin_up[WALT_NR_CPUS];
extern unsigned int sched_capacity_margin_down[WALT_NR_CPUS];
extern cpumask_t asym_cap_sibling_cpus;
extern cpumask_t pipeline_sync_cpus;
extern cpumask_t __read_mostly **cpu_array;
extern int cpu_l2_sibling[WALT_NR_CPUS];
extern void sched_update_nr_prod(int cpu, int enq);
extern unsigned int walt_big_tasks(int cpu);
extern int walt_trailblazer_tasks(int cpu);
extern void walt_rotation_checkpoint(int nr_big);
extern void walt_fill_ta_data(struct core_ctl_notif_data *data);
extern int sched_set_group_id(struct task_struct *p, unsigned int group_id);
extern unsigned int sched_get_group_id(struct task_struct *p);
extern void core_ctl_check(u64 wallclock, u32 wakeup_ctr_sum);
extern int core_ctl_set_cluster_boost(int idx, bool boost);
extern int sched_set_boost(int enable);
extern void walt_boost_init(void);
extern int sched_pause_cpus(struct cpumask *pause_cpus);
extern int sched_unpause_cpus(struct cpumask *unpause_cpus);
extern void walt_kick_cpu(int cpu);
extern void walt_smp_call_newidle_balance(int cpu);
extern bool cpus_halted_by_client(struct cpumask *cpu, enum pause_client client);

extern unsigned int sched_get_cpu_util_pct(int cpu);
extern void sched_update_hyst_times(void);
extern int sched_boost_handler(struct ctl_table *table, int write,
			void __user *buffer, size_t *lenp, loff_t *ppos);
extern int sched_busy_hyst_handler(struct ctl_table *table, int write,
			void __user *buffer, size_t *lenp, loff_t *ppos);
extern u64 walt_sched_clock(void);
extern void walt_init_tg(struct task_group *tg);
extern void walt_init_topapp_tg(struct task_group *tg);
extern void walt_init_foreground_tg(struct task_group *tg);
extern void walt_init_foregroundboost_tg(struct task_group *tg);
extern int register_walt_callback(void);
extern int input_boost_init(void);
extern int core_ctl_init(void);
extern void rebuild_sched_domains(void);

extern atomic64_t walt_irq_work_lastq_ws;
extern unsigned int __read_mostly sched_ravg_window;
extern int min_possible_cluster_id;
extern int max_possible_cluster_id;
extern unsigned int __read_mostly sched_init_task_load_windows;
extern unsigned int __read_mostly sched_load_granule;
extern unsigned long __read_mostly soc_flags;
#define soc_feat(feat)		(soc_flags & feat)
#define soc_feat_set(feat)	(soc_flags |= feat)
#define soc_feat_unset(feat)	(soc_flags &= ~feat)

#define SCHED_IDLE_ENOUGH_DEFAULT 30
#define SCHED_CLUSTER_UTIL_THRES_PCT_DEFAULT 40

extern unsigned int sysctl_sched_idle_enough;
extern unsigned int sysctl_sched_cluster_util_thres_pct;
extern unsigned int sysctl_sched_idle_enough_clust[MAX_CLUSTERS];
extern unsigned int sysctl_sched_cluster_util_thres_pct_clust[MAX_CLUSTERS];

/* 1ms default for 20ms window size scaled to 1024 */
extern unsigned int sysctl_sched_min_task_util_for_boost;
extern unsigned int sysctl_sched_min_task_util_for_uclamp;
/* 0.68ms default for 20ms window size scaled to 1024 */
extern unsigned int sysctl_sched_min_task_util_for_colocation;
extern unsigned int sysctl_sched_busy_hyst_enable_cpus;
extern unsigned int sysctl_sched_busy_hyst;
extern unsigned int sysctl_sched_coloc_busy_hyst_enable_cpus;
extern unsigned int sysctl_sched_coloc_busy_hyst_cpu[WALT_NR_CPUS];
extern unsigned int sysctl_sched_coloc_busy_hyst_max_ms;
extern unsigned int sysctl_sched_coloc_busy_hyst_cpu_busy_pct[WALT_NR_CPUS];
extern unsigned int sysctl_sched_util_busy_hyst_enable_cpus;
extern unsigned int sysctl_sched_util_busy_hyst_cpu[WALT_NR_CPUS];
extern unsigned int sysctl_sched_util_busy_hyst_cpu_util[WALT_NR_CPUS];
extern unsigned int sysctl_sched_boost; /* To/from userspace */
extern unsigned int sysctl_sched_capacity_margin_up[MAX_MARGIN_LEVELS];
extern unsigned int sysctl_sched_capacity_margin_down[MAX_MARGIN_LEVELS];
extern unsigned int sysctl_sched_capacity_margin_up_pct[MAX_MARGIN_LEVELS];
extern unsigned int sysctl_sched_capacity_margin_dn_pct[MAX_MARGIN_LEVELS];
extern unsigned int sysctl_sched_early_up[MAX_MARGIN_LEVELS];
extern unsigned int sysctl_sched_early_down[MAX_MARGIN_LEVELS];
extern unsigned int sched_boost_type; /* currently activated sched boost */
extern enum sched_boost_policy boost_policy;
extern unsigned int sysctl_input_boost_ms;
extern unsigned int sysctl_input_boost_freq[WALT_NR_CPUS];
extern unsigned int sysctl_sched_boost_on_input;
extern unsigned int sysctl_sched_user_hint;
extern unsigned int sysctl_sched_conservative_pl;
extern unsigned int sysctl_sched_hyst_min_coloc_ns;
extern unsigned int sysctl_sched_long_running_rt_task_ms;
extern unsigned int sysctl_ed_boost_pct;
extern unsigned int sysctl_em_inflate_pct;
extern unsigned int sysctl_em_inflate_thres;
extern unsigned int sysctl_sched_heavy_nr;

extern int cpufreq_walt_set_adaptive_freq(unsigned int cpu, unsigned int adaptive_level_1,
					  unsigned int adaptive_low_freq,
					  unsigned int adaptive_high_freq);
extern int cpufreq_walt_get_adaptive_freq(unsigned int cpu, unsigned int *adaptive_level_1,
					  unsigned int *adaptive_low_freq,
					  unsigned int *adaptive_high_freq);
extern int cpufreq_walt_reset_adaptive_freq(unsigned int cpu);

#define WALT_MANY_WAKEUP_DEFAULT 1000
extern unsigned int sysctl_sched_many_wakeup_threshold;
extern unsigned int sysctl_walt_rtg_cfs_boost_prio;
extern __read_mostly unsigned int sysctl_sched_force_lb_enable;
extern const int sched_user_hint_max;
extern unsigned int sysctl_sched_dynamic_tp_enable;
extern unsigned int sysctl_panic_on_walt_bug;
extern unsigned int sysctl_max_freq_partial_halt;
extern unsigned int sysctl_freq_cap[MAX_CLUSTERS];
extern unsigned int high_perf_cluster_freq_cap[MAX_CLUSTERS];
extern unsigned int freq_cap[MAX_FREQ_CAP][MAX_CLUSTERS];
extern unsigned int debugfs_walt_features;
#define walt_feat(feat)		(debugfs_walt_features & feat)
extern int sched_dynamic_tp_handler(struct ctl_table *table, int write,
			void __user *buffer, size_t *lenp, loff_t *ppos);

extern struct list_head cluster_head;
#define for_each_sched_cluster(cluster) \
	list_for_each_entry_rcu(cluster, &cluster_head, list)

static inline struct walt_sched_cluster *cpu_cluster(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return wrq->cluster;
}

static inline u32 cpu_cycles_to_freq(u64 cycles, u64 period)
{
	return div64_u64(cycles, period);
}

static inline unsigned int sched_cpu_legacy_freq(int cpu)
{
	unsigned long curr_cap = arch_scale_freq_capacity(cpu);
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return (curr_cap * (u64) wrq->cluster->max_possible_freq) >>
		SCHED_CAPACITY_SHIFT;
}

extern __read_mostly bool sched_freq_aggr_en;
static inline void walt_enable_frequency_aggregation(bool enable)
{
	sched_freq_aggr_en = enable;
}

#ifndef CONFIG_IRQ_TIME_ACCOUNTING
static inline u64 irq_time_read(int cpu) { return 0; }
#endif

/*Sysctl related interface*/
#define WINDOW_STATS_RECENT		0
#define WINDOW_STATS_MAX		1
#define WINDOW_STATS_MAX_RECENT_AVG	2
#define WINDOW_STATS_AVG		3
#define WINDOW_STATS_INVALID_POLICY	4

extern unsigned int __read_mostly sysctl_sched_coloc_downmigrate_ns;
extern unsigned int __read_mostly sysctl_sched_group_downmigrate_pct;
extern unsigned int __read_mostly sysctl_sched_group_upmigrate_pct;
extern unsigned int __read_mostly sysctl_sched_window_stats_policy;
extern unsigned int sysctl_sched_ravg_window_nr_ticks;
extern unsigned int sysctl_sched_walt_rotate_big_tasks;
extern unsigned int sysctl_sched_task_unfilter_period;
extern unsigned int sysctl_walt_low_latency_task_threshold; /* disabled by default */
extern unsigned int sysctl_sched_sync_hint_enable;
extern unsigned int sysctl_sched_suppress_region2;
extern unsigned int sysctl_sched_skip_sp_newly_idle_lb;
extern unsigned int sysctl_sched_asymcap_boost;

extern void walt_register_sysctl(void);
extern void walt_register_debugfs(void);

extern void walt_config(void);
extern void walt_update_group_thresholds(void);
extern void sched_window_nr_ticks_change(void);
extern unsigned long sched_user_hint_reset_time;
extern struct irq_work walt_migration_irq_work;
extern struct irq_work walt_cpufreq_irq_work;

#define LIB_PATH_LENGTH 512
extern unsigned int cpuinfo_max_freq_cached;
extern char sched_lib_name[LIB_PATH_LENGTH];
extern char sched_lib_task[LIB_PATH_LENGTH];
extern unsigned int sched_lib_mask_force;

extern cpumask_t cpus_for_sbt_pause;
extern unsigned int sysctl_sched_sbt_enable;
extern unsigned int sysctl_sched_sbt_delay_windows;

extern cpumask_t cpus_for_pipeline;
extern bool enable_logging;
extern bool trail_active;
extern bool sustain_active;

/* WALT cpufreq interface */
#define WALT_CPUFREQ_ROLLOVER_BIT		BIT(0)
#define WALT_CPUFREQ_CONTINUE_BIT		BIT(1)
#define WALT_CPUFREQ_IC_MIGRATION_BIT		BIT(2)
#define WALT_CPUFREQ_PL_BIT			BIT(3)
#define WALT_CPUFREQ_EARLY_DET_BIT		BIT(4)
#define WALT_CPUFREQ_BOOST_UPDATE_BIT		BIT(5)
#define WALT_CPUFREQ_ASYM_FIXUP_BIT		BIT(6)
#define WALT_CPUFREQ_SHARED_RAIL_BIT		BIT(7)
#define WALT_CPUFREQ_TRAILBLAZER_BIT		BIT(8)
#define WALT_CPUFREQ_SMART_FREQ_BIT		BIT(9)
#define WALT_CPUFREQ_UCLAMP_BIT			BIT(10)
#define WALT_CPUFREQ_PIPELINE_BUSY_BIT		BIT(11)

/* CPUFREQ_REASON_LOAD is unused. If reasons value is 0, this indicates
 * that no extra features were enforcd, and the frequency alligns with
 * the highest raw workload executing on one of the CPUs within the
 * corresponding cluster
 */
#define CPUFREQ_REASON_LOAD			0
#define CPUFREQ_REASON_BTR_BIT			BIT(0)
#define CPUFREQ_REASON_PL_BIT			BIT(1)
#define CPUFREQ_REASON_EARLY_DET_BIT		BIT(2)
#define CPUFREQ_REASON_RTG_BOOST_BIT		BIT(3)
#define CPUFREQ_REASON_HISPEED_BIT		BIT(4)
#define CPUFREQ_REASON_NWD_BIT			BIT(5)
#define CPUFREQ_REASON_FREQ_AGR_BIT		BIT(6)
#define CPUFREQ_REASON_KSOFTIRQD_BIT		BIT(7)
#define CPUFREQ_REASON_TT_LOAD_BIT		BIT(8)
#define CPUFREQ_REASON_SUH_BIT			BIT(9)
#define CPUFREQ_REASON_ADAPTIVE_LOW_BIT		BIT(10)
#define CPUFREQ_REASON_ADAPTIVE_HIGH_BIT	BIT(11)
#define CPUFREQ_REASON_SMART_FREQ_BIT		BIT(12)
#define CPUFREQ_REASON_HIGH_PERF_CAP_BIT	BIT(13)
#define CPUFREQ_REASON_PARTIAL_HALT_CAP_BIT	BIT(14)
#define CPUFREQ_REASON_TRAILBLAZER_STATE_BIT	BIT(15)
#define CPUFREQ_REASON_TRAILBLAZER_CPU_BIT	BIT(16)
#define CPUFREQ_REASON_ADAPTIVE_LVL_1_BIT	BIT(17)
#define CPUFREQ_REASON_IPC_SMART_FREQ_BIT	BIT(18)
#define CPUFREQ_REASON_UCLAMP_BIT		BIT(19)
#define CPUFREQ_REASON_PIPELINE_BUSY_BIT	BIT(20)

enum sched_boost_policy {
	SCHED_BOOST_NONE,
	SCHED_BOOST_ON_BIG,
	SCHED_BOOST_ON_ALL,
};

struct walt_task_group {
	/*
	 * Controls whether tasks of this cgroup should be colocated with each
	 * other and tasks of other cgroups that have the same flag turned on.
	 */
	bool colocate;
	/*
	 * array indicating whether this task group participates in the
	 * particular boost type
	 */
	bool sched_boost_enable[MAX_NUM_BOOST_TYPE];
};

struct sched_avg_stats {
	int nr;
	int nr_misfit;
	int nr_max;
	int nr_scaled;
};

struct waltgov_callback {
	void (*func)(struct waltgov_callback *cb, u64 time, unsigned int flags);
};

DECLARE_PER_CPU(struct waltgov_callback *, waltgov_cb_data);

static inline void waltgov_add_callback(int cpu, struct waltgov_callback *cb,
			void (*func)(struct waltgov_callback *cb, u64 time,
			unsigned int flags))
{
	if (WARN_ON(!cb || !func))
		return;

	if (WARN_ON(per_cpu(waltgov_cb_data, cpu)))
		return;

	cb->func = func;
	rcu_assign_pointer(per_cpu(waltgov_cb_data, cpu), cb);
}

static inline void waltgov_remove_callback(int cpu)
{
	rcu_assign_pointer(per_cpu(waltgov_cb_data, cpu), NULL);
}

static inline void waltgov_run_callback(struct rq *rq, unsigned int flags)
{
	struct waltgov_callback *cb;

	cb = rcu_dereference_sched(*per_cpu_ptr(&waltgov_cb_data, cpu_of(rq)));
	if (cb)
		cb->func(cb, walt_sched_clock(), flags);
}

extern unsigned long cpu_util_freq_walt(int cpu, struct walt_cpu_load *walt_load,
		unsigned int *reason);
int waltgov_register(void);

extern void walt_lb_init(void);
extern unsigned int walt_rotation_enabled;

extern bool walt_is_idle_task(struct task_struct *p);
/*
 * Returns the current capacity of cpu after applying both
 * cpu and freq scaling.
 */
static inline unsigned long capacity_curr_of(int cpu)
{
	unsigned long max_cap = cpu_rq(cpu)->cpu_capacity_orig;
	unsigned long scale_freq = arch_scale_freq_capacity(cpu);

	return cap_scale(max_cap, scale_freq);
}

static inline unsigned long task_util(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return wts->demand_scaled;
}

static inline unsigned long cpu_util(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);
	u64 walt_cpu_util = wrq->walt_stats.cumulative_runnable_avg_scaled;

	return min_t(unsigned long, walt_cpu_util, capacity_orig_of(cpu));
}

static inline unsigned long cpu_util_cum(int cpu)
{
	return READ_ONCE(cpu_rq(cpu)->cfs.avg.util_avg);
}

/* applying the task threshold for all types of low latency tasks. */
static inline bool walt_low_latency_task(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	if (!wts->low_latency)
		return false;

	if (wts->low_latency & WALT_LOW_LATENCY_BIT_MASK)
		return true;

	/* WALT_LOW_LATENCY_BINDER_BIT and WALT_LOW_LATENCY_PROCFS_BIT remain */
	return (task_util(p) < sysctl_walt_low_latency_task_threshold);
}

static inline bool walt_binder_low_latency_task(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return (wts->low_latency & WALT_LOW_LATENCY_BINDER_BIT) &&
		(task_util(p) < sysctl_walt_low_latency_task_threshold);
}

static inline bool walt_procfs_low_latency_task(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return (wts->low_latency & WALT_LOW_LATENCY_PROCFS_BIT) &&
		(task_util(p) < sysctl_walt_low_latency_task_threshold);
}

static inline bool walt_pipeline_low_latency_task(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return wts->low_latency & WALT_LOW_LATENCY_BIT_MASK;
}

static inline unsigned int walt_get_idle_exit_latency(struct rq *rq)
{
	struct cpuidle_state *idle = idle_get_state(rq);

	if (idle)
		return idle->exit_latency;

	return 0; /* CPU is not idle */
}

static inline bool rt_boost_on_big(void)
{
	return sched_boost_type == FULL_THROTTLE_BOOST ?
			(boost_policy == SCHED_BOOST_ON_BIG) : false;
}

static inline bool is_full_throttle_boost(void)
{
	return sched_boost_type == FULL_THROTTLE_BOOST;
}

static inline bool is_storage_boost(void)
{
	return sched_boost_type == STORAGE_BOOST;
}

static inline bool is_balance_boost(void)
{
	return sched_boost_type == BALANCE_BOOST;
}

static inline bool task_sched_boost(struct task_struct *p)
{
	struct cgroup_subsys_state *css;
	struct task_group *tg;
	bool sched_boost_enabled;
	struct walt_task_group *wtg;

	/* optimization for FT boost, skip looking at tg */
	if (sched_boost_type == FULL_THROTTLE_BOOST)
		return true;

	rcu_read_lock();
	css = task_css(p, cpu_cgrp_id);
	if (!css) {
		rcu_read_unlock();
		return false;
	}
	tg = container_of(css, struct task_group, css);
	wtg = (struct walt_task_group *) tg->android_vendor_data1;
	sched_boost_enabled = wtg->sched_boost_enable[sched_boost_type];
	rcu_read_unlock();

	return sched_boost_enabled;
}

static inline bool task_placement_boost_enabled(struct task_struct *p)
{
	if (likely(boost_policy == SCHED_BOOST_NONE))
		return false;

	return task_sched_boost(p);
}

extern int have_heavy_list;
extern int pipeline_nr;
extern unsigned int sysctl_sched_pipeline_util_thres;
static inline bool pipeline_desired(void)
{
	return sysctl_sched_heavy_nr || sysctl_sched_pipeline_util_thres || pipeline_nr;
}

static inline bool pipeline_in_progress(void)
{
	if (sched_boost_type)
		return false;

	if (pipeline_nr)
		return true;

	if ((sysctl_sched_heavy_nr || sysctl_sched_pipeline_util_thres) && have_heavy_list)
		return true;

	return false;
}

static inline enum sched_boost_policy task_boost_policy(struct task_struct *p)
{
	enum sched_boost_policy policy;

	if (likely(boost_policy == SCHED_BOOST_NONE))
		return SCHED_BOOST_NONE;

	policy = task_sched_boost(p) ? boost_policy : SCHED_BOOST_NONE;
	if (policy == SCHED_BOOST_ON_BIG) {
		/*
		 * Filter out tasks less than min task util threshold
		 * under conservative boost.
		 */
		if (sched_boost_type == CONSERVATIVE_BOOST &&
			task_util(p) <= sysctl_sched_min_task_util_for_boost &&
			!(pipeline_in_progress() && walt_pipeline_low_latency_task(p)))
			policy = SCHED_BOOST_NONE;
		if (sched_boost_type == BALANCE_BOOST &&
			task_util(p) <= sysctl_sched_min_task_util_for_boost)
			policy = SCHED_BOOST_NONE;
	}

	return policy;
}

static inline bool walt_uclamp_boosted(struct task_struct *p)
{
	return soc_feat(SOC_ENABLE_UCLAMP_BOOSTED_BIT) &&
		((uclamp_eff_value(p, UCLAMP_MIN) > 0) &&
		(task_util(p) > sysctl_sched_min_task_util_for_uclamp));
}

static inline unsigned long capacity_of(int cpu)
{
	return cpu_rq(cpu)->cpu_capacity;
}

static inline bool __cpu_overutilized(int cpu, int delta)
{
	return (capacity_orig_of(cpu) * 1024) <
		((cpu_util(cpu) + delta) * sched_capacity_margin_up[cpu]);
}

static inline bool cpu_overutilized(int cpu)
{
	return __cpu_overutilized(cpu, 0);
}

static inline int asym_cap_siblings(int cpu1, int cpu2)
{
	return (cpumask_test_cpu(cpu1, &asym_cap_sibling_cpus) &&
		cpumask_test_cpu(cpu2, &asym_cap_sibling_cpus));
}

/* Is frequency of two cpus synchronized with each other? */
static inline int same_freq_domain(int src_cpu, int dst_cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, src_cpu);

	if (src_cpu == dst_cpu)
		return 1;

	return cpumask_test_cpu(dst_cpu, &wrq->freq_domain_cpumask);
}

static inline unsigned long task_util_est(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return wts->demand_scaled;
}

#ifdef CONFIG_UCLAMP_TASK
static inline unsigned long uclamp_task_util(struct task_struct *p)
{
	return clamp(task_util_est(p),
		     uclamp_eff_value(p, UCLAMP_MIN),
		     uclamp_eff_value(p, UCLAMP_MAX));
}
#else
static inline unsigned long uclamp_task_util(struct task_struct *p)
{
	return task_util_est(p);
}
#endif

static inline int per_task_boost(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	if (wts->boost_period) {
		if (walt_sched_clock() > wts->boost_expires) {
			wts->boost_period = 0;
			wts->boost_expires = 0;
			wts->boost = 0;
		}
	}

	if (!(soc_feat(SOC_ENABLE_PER_TASK_BOOST_ON_MID_BIT)) && (wts->boost == TASK_BOOST_ON_MID))
		return 0;

	return wts->boost;
}

static inline int cluster_first_cpu(struct walt_sched_cluster *cluster)
{
	return cpumask_first(&cluster->cpus);
}

static inline bool hmp_capable(void)
{
	return max_possible_cluster_id != min_possible_cluster_id;
}

static inline bool is_max_possible_cluster_cpu(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return wrq->cluster->id == max_possible_cluster_id;
}

static inline bool is_min_possible_cluster_cpu(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return wrq->cluster->id == min_possible_cluster_id;
}

static inline bool is_min_capacity_cluster(struct walt_sched_cluster *cluster)
{
	return cluster->id == min_possible_cluster_id;
}

/*
 * This is only for tracepoints to print the avg irq load. For
 * task placment considerations, use sched_cpu_high_irqload().
 */
#define SCHED_HIGH_IRQ_TIMEOUT 3
static inline u64 sched_irqload(int cpu)
{
	s64 delta;
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	delta = wrq->window_start - wrq->last_irq_window;
	if (delta < SCHED_HIGH_IRQ_TIMEOUT)
		return wrq->avg_irqload;
	else
		return 0;
}

static inline int sched_cpu_high_irqload(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return wrq->high_irqload;
}

static inline u64
scale_load_to_freq(u64 load, unsigned int src_freq, unsigned int dst_freq)
{
	return div64_u64(load * (u64)src_freq, (u64)dst_freq);
}

static inline unsigned int max_task_load(void)
{
	return sched_ravg_window;
}

static inline int same_cluster(int src_cpu, int dst_cpu)
{
	struct walt_rq *src_wrq = &per_cpu(walt_rq, src_cpu);
	struct walt_rq *dest_wrq = &per_cpu(walt_rq, dst_cpu);

	return src_wrq->cluster == dest_wrq->cluster;
}

static inline bool is_suh_max(void)
{
	return sysctl_sched_user_hint == sched_user_hint_max;
}

#define DEFAULT_CGROUP_COLOC_ID 1
static inline bool walt_should_kick_upmigrate(struct task_struct *p, int cpu)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;
	struct walt_related_thread_group *rtg = wts->grp;

	if (is_suh_max() && rtg && rtg->id == DEFAULT_CGROUP_COLOC_ID &&
			    rtg->skip_min && wts->unfilter)
		return is_min_possible_cluster_cpu(cpu);

	return false;
}

extern bool is_rtgb_active(void);
extern u64 get_rtgb_active_time(void);

static inline unsigned int walt_nr_rtg_high_prio(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return wrq->walt_stats.nr_rtg_high_prio_tasks;
}

static inline bool task_in_related_thread_group(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return (rcu_access_pointer(wts->grp) != NULL);
}

bool walt_halt_check_last(int cpu);
extern struct cpumask __cpu_halt_mask;
extern struct cpumask __cpu_partial_halt_mask;

#define cpu_halt_mask ((struct cpumask *)&__cpu_halt_mask)
#define cpu_partial_halt_mask ((struct cpumask *)&__cpu_partial_halt_mask)

/* a halted cpu must NEVER be used for tasks, as this is the thermal indication to avoid a cpu */
#define cpu_halted(cpu) cpumask_test_cpu((cpu), cpu_halt_mask)

/* a partially halted may be used for helping smaller cpus with small tasks */
#define cpu_partial_halted(cpu) cpumask_test_cpu((cpu), cpu_partial_halt_mask)

#define ASYMCAP_BOOST(cpu)	(sysctl_sched_asymcap_boost && \
				!is_min_possible_cluster_cpu(cpu) && \
				!cpu_partial_halted(cpu))

static bool check_for_higher_capacity(int cpu1, int cpu2)
{
	if (cpu_partial_halted(cpu1) && is_min_possible_cluster_cpu(cpu2))
		return false;

	if (is_min_possible_cluster_cpu(cpu1) && cpu_partial_halted(cpu2))
		return false;

	if (is_max_possible_cluster_cpu(cpu1) && is_min_possible_cluster_cpu(cpu2))
		return capacity_orig_of(cpu1) >= capacity_orig_of(cpu2);

	return capacity_orig_of(cpu1) > capacity_orig_of(cpu2);
}

/* Migration margins for topapp */
extern unsigned int sched_capacity_margin_early_up[WALT_NR_CPUS];
extern unsigned int sched_capacity_margin_early_down[WALT_NR_CPUS];
static inline bool task_fits_capacity(struct task_struct *p,
					int dst_cpu)
{
	unsigned int margin;
	unsigned long capacity = capacity_orig_of(dst_cpu);

	/*
	 * Derive upmigration/downmigrate margin wrt the src/dest CPU.
	 */
	if (check_for_higher_capacity(task_cpu(p), dst_cpu)) {
		margin = sched_capacity_margin_down[dst_cpu];
		if (task_in_related_thread_group(p))
			margin = max(margin, sched_capacity_margin_early_down[dst_cpu]);
	} else {
		margin = sched_capacity_margin_up[task_cpu(p)];
		if (task_in_related_thread_group(p))
			margin = max(margin, sched_capacity_margin_early_up[task_cpu(p)]);
	}

	return capacity * 1024 > uclamp_task_util(p) * margin;
}

extern int pipeline_fits_smaller_cpus(struct task_struct *p);
static inline bool task_fits_max(struct task_struct *p, int dst_cpu)
{
	unsigned long task_boost = per_task_boost(p);
	cpumask_t other_cluster;
	int ret = -1;

	/*
	 * If a task is affined only to cpus of cluster then it cannot be a
	 * misfit
	 */
	cpumask_andnot(&other_cluster, cpu_possible_mask, &cpu_cluster(dst_cpu)->cpus);
	if (!cpumask_intersects(&other_cluster, p->cpus_ptr))
		return true;

	if (is_max_possible_cluster_cpu(dst_cpu))
		return true;

	ret = pipeline_fits_smaller_cpus(p);
	if (ret == 0)
		return false;
	else if (ret == 1)
		return true;

	if (is_min_possible_cluster_cpu(dst_cpu)) {
		if (task_boost_policy(p) == SCHED_BOOST_ON_BIG ||
				task_boost > 0 ||
				walt_uclamp_boosted(p) ||
				walt_should_kick_upmigrate(p, dst_cpu))
			return false;
	} else { /* mid cap cpu */
		if (task_boost > TASK_BOOST_ON_MID)
			return false;
		if (!task_in_related_thread_group(p) && p->prio >= 124)
			/* a non topapp low prio task fits on gold */
			return true;
	}

	return task_fits_capacity(p, dst_cpu);
}

extern struct sched_avg_stats *sched_get_nr_running_avg(void);
extern unsigned int sched_get_cluster_util_pct(struct walt_sched_cluster *cluster);
extern void sched_update_hyst_times(void);

extern void walt_rt_init(void);
extern void walt_cfs_init(void);
extern void walt_halt_init(void);
extern void walt_mvp_lock_ordering_init(void);
extern void walt_fixup_init(void);
extern int walt_find_energy_efficient_cpu(struct task_struct *p, int prev_cpu,
					int sync, int sibling_count_hint);
extern int walt_find_cluster_packing_cpu(int start_cpu);
extern bool walt_choose_packing_cpu(int packing_cpu, struct task_struct *p);
extern void walt_cycle_counter_init(void);
extern u64 walt_cpu_cycle_counter(int cpu, u64 wc);

static inline unsigned int cpu_max_possible_freq(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return wrq->cluster->max_possible_freq;
}

static inline unsigned int cpu_max_freq(int cpu)
{
	return mult_frac(cpu_max_possible_freq(cpu), capacity_orig_of(cpu),
			 arch_scale_cpu_capacity(cpu));
}

static inline unsigned int task_load(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return wts->demand;
}

static inline bool task_rtg_high_prio(struct task_struct *p)
{
	return task_in_related_thread_group(p) &&
		(p->prio <= sysctl_walt_rtg_cfs_boost_prio);
}

static inline struct walt_related_thread_group
*task_related_thread_group(struct task_struct *p)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return rcu_dereference(wts->grp);
}

static inline bool walt_get_rtg_status(struct task_struct *p)
{
	struct walt_related_thread_group *grp;
	bool ret = false;

	if (!soc_feat(SOC_ENABLE_COLOCATION_PLACEMENT_BOOST_BIT))
		return ret;

	rcu_read_lock();

	grp = task_related_thread_group(p);
	if (grp)
		ret = grp->skip_min;

	rcu_read_unlock();

	return ret;
}

#define CPU_RESERVED			BIT(0)
#define CPU_FIRST_ENQ_IN_WINDOW		BIT(1)
static inline int is_reserved(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return test_bit(CPU_RESERVED, &wrq->walt_flags);
}

static inline int mark_reserved(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return test_and_set_bit(CPU_RESERVED, &wrq->walt_flags);
}

static inline void clear_reserved(int cpu)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	clear_bit(CPU_RESERVED, &wrq->walt_flags);
}

static inline bool is_cpu_flag_set(int cpu, unsigned int bit)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	return test_bit(bit, &wrq->walt_flags);
}

static inline void set_cpu_flag(int cpu, unsigned int bit, bool set)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, cpu);

	if (set)
		set_bit(bit, &wrq->walt_flags);
	else
		clear_bit(bit, &wrq->walt_flags);
}

static inline void walt_irq_work_queue(struct irq_work *work)
{
	if (likely(cpu_online(raw_smp_processor_id())))
		irq_work_queue(work);
	else
		irq_work_queue_on(work, cpumask_any(cpu_online_mask));
}

static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
{
	return css ? container_of(css, struct task_group, css) : NULL;
}

/*
 * The policy of a RT boosted task (via PI mutex) still indicates it is
 * a fair task, so use prio check as well. The prio check alone is not
 * sufficient since idle task also has 120 priority.
 */
static inline bool walt_fair_task(struct task_struct *p)
{
	return p->prio >= MAX_RT_PRIO && !walt_is_idle_task(p);
}

extern int sched_long_running_rt_task_ms_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos);

static inline void walt_flag_set(struct task_struct *p, unsigned int feature, bool set)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	if (set)
		wts->flags |= feature;
	else
		wts->flags &= ~feature;
}

static inline bool walt_flag_test(struct task_struct *p, unsigned int feature)
{
	struct walt_task_struct *wts = (struct walt_task_struct *) p->android_vendor_data1;

	return wts->flags & feature;
}

#define WALT_MVP_SLICE		3000000U
#define WALT_MVP_LIMIT		(4 * WALT_MVP_SLICE)

/* higher number, better priority */
#define WALT_RTG_MVP		0
#define WALT_BINDER_MVP		1
#define WALT_TASK_BOOST_MVP	2
#define WALT_LL_PIPE_MVP	3

#define WALT_NOT_MVP		-1

#define is_mvp(wts) (wts->mvp_prio != WALT_NOT_MVP)
void walt_cfs_enqueue_task(struct rq *rq, struct task_struct *p);
void walt_cfs_dequeue_task(struct rq *rq, struct task_struct *p);
void walt_cfs_tick(struct rq *rq);
void walt_lb_tick(struct rq *rq);

extern __read_mostly unsigned int walt_scale_demand_divisor;

static inline u64 scale_time_to_util(u64 d)
{
	/*
	 * The denominator at most could be (8 * tick_size) >> SCHED_CAPACITY_SHIFT,
	 * a value that easily fits a 32bit integer.
	 */
	do_div(d, walt_scale_demand_divisor);
	return d;
}

void create_util_to_cost(void);
struct compute_energy_output {
	unsigned long	sum_util[MAX_CLUSTERS];
	unsigned long	max_util[MAX_CLUSTERS];
	unsigned long	cost[MAX_CLUSTERS];
	unsigned int	cluster_first_cpu[MAX_CLUSTERS];
};

static inline bool is_state1(void)
{
	struct cpumask local_mask = { CPU_BITS_NONE };

	if (!cpumask_weight(&part_haltable_cpus))
		return false;

	cpumask_or(&local_mask, cpu_partial_halt_mask, cpu_halt_mask);
	return cpumask_subset(&part_haltable_cpus, &local_mask);
}

/* determine if this task should be allowed to use a partially halted cpu */
static inline bool task_reject_partialhalt_cpu(struct task_struct *p, int cpu)
{
	if (p->prio < MAX_RT_PRIO)
		return false;

	if (cpu_partial_halted(cpu) && !task_fits_capacity(p, 0))
		return true;

	return false;
}

/* walt_find_and_choose_cluster_packing_cpu - Return a packing_cpu choice common for this cluster.
 * @start_cpu:  The cpu from the cluster to choose from
 *
 * If the cluster has a 32bit capable cpu return it regardless
 * of whether it is halted or not.
 *
 * If the cluster does not have a 32 bit capable cpu, find the
 * first unhalted, active cpu in this cluster.
 *
 * Returns -1 if packing_cpu if not found or is unsuitable to be packed on  to
 * Returns a valid cpu number if packing_cpu is found and is usable
 */
static inline int walt_find_and_choose_cluster_packing_cpu(int start_cpu, struct task_struct *p)
{
	struct walt_rq *wrq = &per_cpu(walt_rq, start_cpu);
	struct walt_sched_cluster *cluster = wrq->cluster;
	cpumask_t unhalted_cpus;
	int packing_cpu;

	/* if idle_enough feature is not enabled */
	if (!sysctl_sched_idle_enough_clust[cluster->id])
		return -1;
	if (!sysctl_sched_cluster_util_thres_pct_clust[cluster->id])
		return -1;

	/* find all unhalted active cpus */
	cpumask_andnot(&unhalted_cpus, cpu_active_mask, cpu_halt_mask);

	/* find all unhalted active cpus in this cluster */
	cpumask_and(&unhalted_cpus, &unhalted_cpus, &cluster->cpus);

	if (is_compat_thread(task_thread_info(p)))
		/* try to find a packing cpu within 32 bit subset */
		cpumask_and(&unhalted_cpus, &unhalted_cpus, system_32bit_el0_cpumask());

	/* return the first found unhalted, active cpu, in this cluster */
	packing_cpu = cpumask_first(&unhalted_cpus);

	/* packing cpu must be a valid cpu for runqueue lookup */
	if (packing_cpu >= nr_cpu_ids)
		return -1;

	/* if cpu is not allowed for this task */
	if (!cpumask_test_cpu(packing_cpu, p->cpus_ptr))
		return -1;

	/* if cluster util is high */
	if (sched_get_cluster_util_pct(cluster) >=
	    sysctl_sched_cluster_util_thres_pct_clust[cluster->id])
		return -1;

	/* if cpu utilization is high */
	if (cpu_util(packing_cpu) >= sysctl_sched_idle_enough_clust[cluster->id])
		return -1;

	/* don't pack big tasks */
	if (task_util(p) >= sysctl_sched_idle_enough_clust[cluster->id])
		return -1;

	if (task_reject_partialhalt_cpu(p, packing_cpu))
		return -1;

	/* don't pack if running at a freq higher than 43.9pct of its fmax */
	if (arch_scale_freq_capacity(packing_cpu) > 450)
		return -1;

	/* the packing cpu can be used, so pack! */
	return packing_cpu;
}

extern void update_smart_freq_capacities(void);
extern void update_cpu_capacity_helper(int cpu);
extern void smart_freq_update_for_all_cluster(u64 wallclock, uint32_t reasons);
extern void smart_freq_update_reason_common(u64 window_start, int nr_big, u32 wakeup_ctr_sum);
extern void smart_freq_init(const char *name);
extern unsigned int get_cluster_ipc_level_freq(int curr_cpu, u64 time);
extern void update_smart_freq_capacities_one_cluster(struct walt_sched_cluster *cluster);

extern int add_pipeline(struct walt_task_struct *wts);
extern int remove_pipeline(struct walt_task_struct *wts);

extern void walt_task_dump(struct task_struct *p);
extern void walt_rq_dump(int cpu);
extern void walt_dump(void);
extern int in_sched_bug;

extern struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
				 struct task_struct *p, int dest_cpu);

DECLARE_PER_CPU(u64, rt_task_arrival_time);
extern int walt_get_mvp_task_prio(struct task_struct *p);
extern void walt_cfs_deactivate_mvp_task(struct rq *rq, struct task_struct *p);

void inc_rq_walt_stats(struct rq *rq, struct task_struct *p);
void dec_rq_walt_stats(struct rq *rq, struct task_struct *p);

extern bool is_obet;
extern int oscillate_cpu;
extern int oscillate_period_ns;
extern bool should_oscillate(unsigned int busy_cpu, int *no_oscillate_reason);
extern bool now_is_sbt;
extern bool is_sbt_or_oscillate(void);

extern unsigned int sysctl_sched_walt_core_util[WALT_NR_CPUS];
extern unsigned int sysctl_pipeline_busy_boost_pct;

enum WALT_DEBUG_FEAT {
	WALT_BUG_UPSTREAM,
	WALT_BUG_WALT,
	WALT_BUG_UNUSED,

	/* maximum 4 entries allowed */
	WALT_DEBUG_FEAT_NR,
};

#define WALT_PANIC(condition)				\
({							\
	if (unlikely(!!(condition)) && !in_sched_bug) {	\
		in_sched_bug = 1;			\
		walt_dump();				\
		BUG_ON(condition);			\
	}						\
})

/* the least signifcant byte is the bitmask for features and printk */
#define WALT_PANIC_SENTINEL	0x4544DE00

#define walt_debug_bitmask_panic(x) (1UL << x)
#define walt_debug_bitmask_print(x) (1UL << (x + WALT_DEBUG_FEAT_NR))

/* setup initial values, bug and print on upstream and walt, ignore noncritical */
#define walt_debug_initial_values()			\
	(WALT_PANIC_SENTINEL |				\
	 walt_debug_bitmask_panic(WALT_BUG_UPSTREAM) |	\
	 walt_debug_bitmask_print(WALT_BUG_UPSTREAM) |	\
	 walt_debug_bitmask_panic(WALT_BUG_WALT) |	\
	 walt_debug_bitmask_print(WALT_BUG_WALT))

/* least significant nibble is the bug feature itself */
#define walt_debug_feat_panic(x) (!!(sysctl_panic_on_walt_bug & (1UL << x)))

/* 2nd least significant nibble is the print capability */
#define walt_debug_feat_print(x) (!!(sysctl_panic_on_walt_bug & (1UL << (x + WALT_DEBUG_FEAT_NR))))

/* return true if the sentinel is set, regardless of feature set */
static inline bool is_walt_sentinel(void)
{
	if (unlikely((sysctl_panic_on_walt_bug & 0xFFFFFF00) == WALT_PANIC_SENTINEL))
		return true;
	return false;
}

/* if the sentinel is properly set, print and/or panic as configured */
#define WALT_BUG(feat, p, format, args...)					\
({										\
	if (is_walt_sentinel()) {						\
		if (walt_debug_feat_print(feat))				\
			printk_deferred("WALT-BUG " format, args);		\
		if (walt_debug_feat_panic(feat)) {				\
			if (p)							\
				walt_task_dump(p);				\
			WALT_PANIC(1);						\
		}								\
	}									\
})

static inline void walt_lockdep_assert(int cond, int cpu, struct task_struct *p)
{
	if (!cond) {
		pr_err("LOCKDEP: %pS %ps %ps %ps\n",
		       __builtin_return_address(0),
		       __builtin_return_address(1),
		       __builtin_return_address(2),
		       __builtin_return_address(3));
		WALT_BUG(WALT_BUG_WALT, p,
			 "running_cpu=%d cpu_rq=%d cpu_rq lock not held",
			 raw_smp_processor_id(), cpu);
	}
}

#ifdef CONFIG_LOCKDEP
#define walt_lockdep_assert_held(l, cpu, p)				\
	walt_lockdep_assert(lockdep_is_held(l) != LOCK_STATE_NOT_HELD, cpu, p)
#else
#define walt_lockdep_assert_held(l, cpu, p) do { (void)(l); } while (0)
#endif

#define walt_lockdep_assert_rq(rq, p)			\
	walt_lockdep_assert_held(&rq->__lock, cpu_of(rq), p)

extern bool pipeline_check(struct walt_rq *wrq);
extern void pipeline_rearrange(struct walt_rq *wrq, bool need_assign_heavy);
extern bool enable_load_sync(int cpu);
extern struct walt_related_thread_group *lookup_related_thread_group(unsigned int group_id);
extern bool prev_is_sbt;
extern unsigned int sysctl_sched_pipeline_special;
extern struct task_struct *pipeline_special_task;
extern void remove_special_task(void);
extern void set_special_task(struct task_struct *pipeline_special_local);
#define MAX_NR_PIPELINE 3
/* smart freq */
#define SMART_FREQ_LEGACY_TUPLE_SIZE		3
#define SMART_FREQ_IPC_TUPLE_SIZE		3

extern char reason_dump[1024];
extern void update_smart_freq_capacities_one_cluster(struct walt_sched_cluster *cluster);
extern int sched_smart_freq_legacy_dump_handler(struct ctl_table *table, int write,
					      void __user *buffer, size_t *lenp, loff_t *ppos);
extern int sched_smart_freq_ipc_dump_handler(struct ctl_table *table, int write,
					   void __user *buffer, size_t *lenp, loff_t *ppos);
extern unsigned int sysctl_ipc_freq_levels_cluster0[SMART_FMAX_IPC_MAX];
extern unsigned int sysctl_ipc_freq_levels_cluster1[SMART_FMAX_IPC_MAX];
extern unsigned int sysctl_ipc_freq_levels_cluster2[SMART_FMAX_IPC_MAX];
extern unsigned int sysctl_ipc_freq_levels_cluster3[SMART_FMAX_IPC_MAX];
extern int sched_smart_freq_ipc_handler(struct ctl_table *table, int write,
				      void __user *buffer, size_t *lenp,
				      loff_t *ppos);

extern u8 smart_freq_legacy_reason_hyst_ms[LEGACY_SMART_FREQ][WALT_NR_CPUS];
extern void update_smart_freq_legacy_reason_hyst_time(struct walt_sched_cluster *cluster);

/* frequent yielder */
#define MAX_YIELD_CNT_PER_TASK_THR		25
#define	YIELD_INDUCED_SLEEP			BIT(7)
#define YIELD_CNT_MASK				0x7F
#define MAX_YIELD_CNT_GLOBAL_THR		8000
#define YIELD_WINDOW_SIZE_USEC			(16ULL * USEC_PER_MSEC)
#define YIELD_WINDOW_SIZE_NSEC			(YIELD_WINDOW_SIZE_USEC * NSEC_PER_USEC)
#define	YIELD_GRACE_PERIOD_NSEC			(4ULL * NSEC_PER_MSEC)
#define MIN_CONTIGUOUS_YIELDING_WINDOW		3
#define YIELD_SLEEP_TIME_USEC			250
#define MAX_YIELD_SLEEP_CNT_GLOBAL_THR		(YIELD_WINDOW_SIZE_USEC /		\
								YIELD_SLEEP_TIME_USEC / 2)
extern u8 contiguous_yielding_windows;
#define NUM_PIPELINE_BUSY_THRES 3
extern unsigned int sysctl_sched_lrpb_active_ms[NUM_PIPELINE_BUSY_THRES];
#define NUM_LOAD_SYNC_SETTINGS 3
extern unsigned int sysctl_cluster01_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster01_load_sync_60fps[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster02_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster03_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster10_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster10_load_sync_60fps[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster12_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster13_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster20_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster21_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster23_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster30_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster31_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int sysctl_cluster32_load_sync[NUM_LOAD_SYNC_SETTINGS];
extern unsigned int load_sync_util_thres[MAX_CLUSTERS][MAX_CLUSTERS];
extern unsigned int load_sync_util_thres_60fps[MAX_CLUSTERS][MAX_CLUSTERS];
extern unsigned int load_sync_low_pct[MAX_CLUSTERS][MAX_CLUSTERS];
extern unsigned int load_sync_low_pct_60fps[MAX_CLUSTERS][MAX_CLUSTERS];
extern unsigned int load_sync_high_pct[MAX_CLUSTERS][MAX_CLUSTERS];
extern unsigned int load_sync_high_pct_60fps[MAX_CLUSTERS][MAX_CLUSTERS];
extern unsigned int sysctl_pipeline_special_task_util_thres;
extern unsigned int sysctl_pipeline_non_special_task_util_thres;
extern unsigned int sysctl_pipeline_pin_thres_low_pct;
extern unsigned int sysctl_pipeline_pin_thres_high_pct;
#endif /* _WALT_H */