samsung_sm8750/android_kernel_samsung_sm8750
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
lineage-22.2
android_kernel_samsung_sm8750/drivers/soc/qcom/rpmh-rsc.c
SaschaNes 94a2807785 add samsung opensource kernel
2025-08-12 22:16:57 +02:00

1940 lines
54 KiB
C
Executable File
Raw Permalink Blame History

// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
* Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved.
*/
#define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME
#include <linux/atomic.h>
#include <linux/cpu_pm.h>
#include <linux/delay.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/iopoll.h>
#include <linux/kernel.h>
#include <linux/ktime.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/notifier.h>
#include <linux/of.h>
#include <linux/of_irq.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/pm_domain.h>
#include <linux/pm_runtime.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/wait.h>
#include <clocksource/arm_arch_timer.h>
#include <soc/qcom/cmd-db.h>
#include <soc/qcom/tcs.h>
#include <soc/qcom/crm.h>
#include <dt-bindings/soc/qcom,rpmh-rsc.h>
#include "rpmh-internal.h"
#define CREATE_TRACE_POINTS
#include "trace-rpmh.h"
#if IS_ENABLED(CONFIG_IPC_LOGGING)
#include <linux/ipc_logging.h>
#endif
#define RSC_DRV_IPC_LOG_SIZE 2
/* DRV ID Register */
#define RSC_DRV_ID 0
#define MAJOR_VER_MASK 0xFF
#define MAJOR_VER_SHIFT 16
#define MINOR_VER_MASK 0xFF
#define MINOR_VER_SHIFT 8
enum {
RSC_DRV_TCS_OFFSET,
RSC_DRV_CMD_OFFSET,
DRV_SOLVER_CONFIG,
DRV_PRNT_CHLD_CONFIG,
RSC_DRV_IRQ_ENABLE,
RSC_DRV_IRQ_STATUS,
RSC_DRV_IRQ_CLEAR,
RSC_DRV_CMD_WAIT_FOR_CMPL,
RSC_DRV_CONTROL,
RSC_DRV_STATUS,
RSC_DRV_CMD_ENABLE,
RSC_DRV_CMD_MSGID,
RSC_DRV_CMD_ADDR,
RSC_DRV_CMD_DATA,
RSC_DRV_CMD_STATUS,
RSC_DRV_CMD_RESP_DATA,
/* DRV channel Registers */
RSC_DRV_CHN_TCS_TRIGGER,
RSC_DRV_CHN_TCS_COMPLETE,
RSC_DRV_CHN_SEQ_BUSY,
RSC_DRV_CHN_SEQ_PC,
RSC_DRV_CHN_UPDATE,
RSC_DRV_CHN_BUSY,
RSC_DRV_CHN_EN,
};
/* DRV HW Solver Configuration Information Register */
#define DRV_HW_SOLVER_MASK 1
#define DRV_HW_SOLVER_SHIFT 24
/* DRV TCS Configuration Information Register */
#define DRV_NUM_TCS_MASK 0x3F
#define DRV_NUM_TCS_SHIFT 6
#define DRV_NCPT_MASK 0x1F
#define DRV_NCPT_SHIFT 27
/* Offsets for CONTROL TCS Registers */
#define RSC_DRV_CTL_TCS_DATA_HI 0x38
#define RSC_DRV_CTL_TCS_DATA_HI_MASK 0xFFFFFF
#define RSC_DRV_CTL_TCS_DATA_HI_VALID BIT(31)
#define RSC_DRV_CTL_TCS_DATA_LO 0x40
#define RSC_DRV_CTL_TCS_DATA_LO_MASK 0xFFFFFFFF
#define RSC_DRV_CTL_TCS_DATA_SIZE 32
#define TCS_AMC_MODE_ENABLE BIT(16)
#define TCS_AMC_MODE_TRIGGER BIT(24)
/* TCS CMD register bit mask */
#define CMD_MSGID_LEN 8
#define CMD_MSGID_RESP_REQ BIT(8)
#define CMD_MSGID_WRITE BIT(16)
#define CMD_STATUS_ISSUED BIT(8)
#define CMD_STATUS_COMPL BIT(16)
/* Offsets for DRV channel status register */
#define CH0_CHN_BUSY BIT(0)
#define CH1_CHN_BUSY BIT(1)
#define CH0_WAKE_TCS_STATUS BIT(0)
#define CH0_SLEEP_TCS_STATUS BIT(1)
#define CH1_WAKE_TCS_STATUS BIT(2)
#define CH1_SLEEP_TCS_STATUS BIT(3)
#define CH_CLEAR_STATUS BIT(31)
#define ACCL_TYPE(addr) ((addr >> 16) & 0xF)
#define VREG_ADDR(addr) (addr & ~0xF)
#define MAX_RSC_COUNT 5
enum {
HW_ACCL_CLK = 0x3,
HW_ACCL_VREG,
HW_ACCL_BUS,
};
static const char * const accl_str[] = {
"", "", "", "CLK", "VREG", "BUS",
};
static LIST_HEAD(rpmh_rsc_dev_list);
static struct rsc_drv *__rsc_drv[MAX_RSC_COUNT];
static int __rsc_count;
bool rpmh_standalone;
/*
* Here's a high level overview of how all the registers in RPMH work
* together:
*
* - The main rpmh-rsc address is the base of a register space that can
* be used to find overall configuration of the hardware
* (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register
* space are all the TCS blocks. The offset of the TCS blocks is
* specified in the device tree by "qcom,tcs-offset" and used to
* compute tcs_base.
* - TCS blocks come one after another. Type, count, and order are
* specified by the device tree as "qcom,tcs-config".
* - Each TCS block has some registers, then space for up to 16 commands.
* Note that though address space is reserved for 16 commands, fewer
* might be present. See ncpt (num cmds per TCS).
*
* Here's a picture:
*
* +---------------------------------------------------+
* |RSC |
* | ctrl |
* | |
* | Drvs: |
* | +-----------------------------------------------+ |
* | |DRV0 | |
* | | ctrl/config | |
* | | IRQ | |
* | | | |
* | | TCSes: | |
* | | +------------------------------------------+ | |
* | | |TCS0 | | | | | | | | | | | | | | |
* | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
* | | | | | | | | | | | | | | | | | |
* | | +------------------------------------------+ | |
* | | +------------------------------------------+ | |
* | | |TCS1 | | | | | | | | | | | | | | |
* | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
* | | | | | | | | | | | | | | | | | |
* | | +------------------------------------------+ | |
* | | +------------------------------------------+ | |
* | | |TCS2 | | | | | | | | | | | | | | |
* | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | |
* | | | | | | | | | | | | | | | | | |
* | | +------------------------------------------+ | |
* | | ...... | |
* | +-----------------------------------------------+ |
* | +-----------------------------------------------+ |
* | |DRV1 | |
* | | (same as DRV0) | |
* | +-----------------------------------------------+ |
* | ...... |
* +---------------------------------------------------+
*/
#define USECS_TO_CYCLES(time_usecs) \
xloops_to_cycles((time_usecs) * 0x10C7UL)
static inline unsigned long xloops_to_cycles(u64 xloops)
{
return (xloops * loops_per_jiffy * HZ) >> 32;
}
static u32 rpmh_rsc_reg_offset_ver_2_7[] = {
[RSC_DRV_TCS_OFFSET] = 672,
[RSC_DRV_CMD_OFFSET] = 20,
[DRV_SOLVER_CONFIG] = 0x04,
[DRV_PRNT_CHLD_CONFIG] = 0x0C,
[RSC_DRV_IRQ_ENABLE] = 0x00,
[RSC_DRV_IRQ_STATUS] = 0x04,
[RSC_DRV_IRQ_CLEAR] = 0x08,
[RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10,
[RSC_DRV_CONTROL] = 0x14,
[RSC_DRV_STATUS] = 0x18,
[RSC_DRV_CMD_ENABLE] = 0x1C,
[RSC_DRV_CMD_MSGID] = 0x30,
[RSC_DRV_CMD_ADDR] = 0x34,
[RSC_DRV_CMD_DATA] = 0x38,
[RSC_DRV_CMD_STATUS] = 0x3C,
[RSC_DRV_CMD_RESP_DATA] = 0x40,
[RSC_DRV_CHN_SEQ_BUSY] = 0x0,
[RSC_DRV_CHN_SEQ_PC] = 0x0,
[RSC_DRV_CHN_TCS_TRIGGER] = 0x0,
[RSC_DRV_CHN_TCS_COMPLETE] = 0x0,
[RSC_DRV_CHN_UPDATE] = 0x0,
[RSC_DRV_CHN_BUSY] = 0x0,
[RSC_DRV_CHN_EN] = 0x0,
};
static u32 rpmh_rsc_reg_offset_ver_3_0[] = {
[RSC_DRV_TCS_OFFSET] = 672,
[RSC_DRV_CMD_OFFSET] = 24,
[DRV_SOLVER_CONFIG] = 0x04,
[DRV_PRNT_CHLD_CONFIG] = 0x0C,
[RSC_DRV_IRQ_ENABLE] = 0x00,
[RSC_DRV_IRQ_STATUS] = 0x04,
[RSC_DRV_IRQ_CLEAR] = 0x08,
[RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20,
[RSC_DRV_CONTROL] = 0x24,
[RSC_DRV_STATUS] = 0x28,
[RSC_DRV_CMD_ENABLE] = 0x2C,
[RSC_DRV_CMD_MSGID] = 0x34,
[RSC_DRV_CMD_ADDR] = 0x38,
[RSC_DRV_CMD_DATA] = 0x3C,
[RSC_DRV_CMD_STATUS] = 0x40,
[RSC_DRV_CMD_RESP_DATA] = 0x44,
[RSC_DRV_CHN_SEQ_BUSY] = 0x464,
[RSC_DRV_CHN_SEQ_PC] = 0x468,
[RSC_DRV_CHN_TCS_TRIGGER] = 0x490,
[RSC_DRV_CHN_TCS_COMPLETE] = 0x494,
[RSC_DRV_CHN_UPDATE] = 0x498,
[RSC_DRV_CHN_BUSY] = 0x49C,
[RSC_DRV_CHN_EN] = 0x4A0,
};
static u32 rpmh_rsc_reg_offset_ver_3_0_hw_channel[] = {
[RSC_DRV_TCS_OFFSET] = 336,
[RSC_DRV_CMD_OFFSET] = 24,
[DRV_SOLVER_CONFIG] = 0x04,
[DRV_PRNT_CHLD_CONFIG] = 0x0C,
[RSC_DRV_IRQ_ENABLE] = 0x00,
[RSC_DRV_IRQ_STATUS] = 0x04,
[RSC_DRV_IRQ_CLEAR] = 0x08,
[RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20,
[RSC_DRV_CONTROL] = 0x24,
[RSC_DRV_STATUS] = 0x28,
[RSC_DRV_CMD_ENABLE] = 0x2C,
[RSC_DRV_CMD_MSGID] = 0x34,
[RSC_DRV_CMD_ADDR] = 0x38,
[RSC_DRV_CMD_DATA] = 0x3C,
[RSC_DRV_CMD_STATUS] = 0x40,
[RSC_DRV_CMD_RESP_DATA] = 0x44,
[RSC_DRV_CHN_SEQ_BUSY] = 0x464,
[RSC_DRV_CHN_SEQ_PC] = 0x468,
[RSC_DRV_CHN_TCS_TRIGGER] = 0x490,
[RSC_DRV_CHN_TCS_COMPLETE] = 0x494,
[RSC_DRV_CHN_UPDATE] = 0x498,
[RSC_DRV_CHN_BUSY] = 0x49C,
[RSC_DRV_CHN_EN] = 0x4A0,
};
static inline void __iomem *
tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id)
{
if (!drv->tcs_distance)
return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg;
return drv->tcs_base + drv->tcs_distance * tcs_id + reg;
}
static inline void __iomem *
tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id)
{
return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id;
}
static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
int cmd_id)
{
return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
}
static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id)
{
return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id));
}
static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id,
int cmd_id, u32 data)
{
writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id));
}
static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id,
u32 data)
{
writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id));
}
static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id,
u32 data)
{
int i;
writel(data, tcs_reg_addr(drv, reg, tcs_id));
/*
* Wait until we read back the same value. Use a counter rather than
* ktime for timeout since this may be called after timekeeping stops.
*/
for (i = 0; i < USEC_PER_SEC; i++) {
if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data)
return;
udelay(1);
}
pr_err("%s: error writing %#x to %d:%#x\n", drv->name,
data, tcs_id, reg);
}
/**
* tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake).
* @drv: The RSC controller.
* @type: SLEEP_TCS or WAKE_TCS
* @ch: Channel number
*
* This will clear the "slots" variable of the given tcs_group and also
* tell the hardware to forget about all entries.
*
* The caller must ensure that no other RPMH actions are happening when this
* function is called, since otherwise the device may immediately become
* used again even before this function exits.
*/
static void tcs_invalidate(struct rsc_drv *drv, int type, int ch)
{
int m;
struct tcs_group *tcs = &drv->ch[ch].tcs[type];
/* Caller ensures nobody else is running so no lock */
if (bitmap_empty(tcs->slots, tcs->ncpt * tcs->num_tcs))
return;
for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) {
write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0);
write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], m, 0);
}
bitmap_zero(tcs->slots, tcs->ncpt * tcs->num_tcs);
}
/**
* rpmh_rsc_get_channel() - Get the Unused channel to send data on.
* @drv: The RSC controller.
*
* Return: 0 on success, else -error.
*/
int rpmh_rsc_get_channel(struct rsc_drv *drv)
{
int chn_update, chn_busy;
if (drv->num_channels == 1)
return CH0;
/* Select Unused channel */
do {
chn_update = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_UPDATE]);
chn_busy = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_BUSY]);
} while (chn_busy != chn_update);
if (chn_busy & CH0_CHN_BUSY)
return CH1;
else if (chn_busy & CH1_CHN_BUSY)
return CH0;
else
return -EBUSY;
}
/**
* rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes.
* @drv: The RSC controller.
* @ch: Channel number
*
* The caller must ensure that no other RPMH actions are happening when this
* function is called, since otherwise the device may immediately become
* used again even before this function exits.
*/
void rpmh_rsc_invalidate(struct rsc_drv *drv, int ch)
{
tcs_invalidate(drv, SLEEP_TCS, ch);
tcs_invalidate(drv, WAKE_TCS, ch);
}
/**
* get_tcs_for_msg() - Get the tcs_group used to send the given message.
* @drv: The RSC controller.
* @msg: The message we want to send.
*
* This is normally pretty straightforward except if we are trying to send
* an ACTIVE_ONLY message but don't have any active_only TCSes.
*
* Return: A pointer to a tcs_group or an ERR_PTR.
*/
static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv,
enum rpmh_state state,
int ch)
{
int type;
struct tcs_group *tcs;
switch (state) {
case RPMH_ACTIVE_ONLY_STATE:
type = ACTIVE_TCS;
break;
case RPMH_WAKE_ONLY_STATE:
type = WAKE_TCS;
break;
case RPMH_SLEEP_STATE:
type = SLEEP_TCS;
break;
default:
return ERR_PTR(-EINVAL);
}
/*
* If we are making an active request on a RSC that does not have a
* dedicated TCS for active state use, then re-purpose a wake TCS to
* send active votes. This is safe because we ensure any active-only
* transfers have finished before we use it (maybe by running from
* the last CPU in PM code).
*/
tcs = &drv->ch[ch].tcs[type];
if (state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs)
tcs = &drv->ch[ch].tcs[WAKE_TCS];
return tcs;
}
/**
* get_req_from_tcs() - Get a stashed request that was xfering on the given TCS.
* @drv: The RSC controller.
* @tcs_id: The global ID of this TCS.
*
* For ACTIVE_ONLY transfers we want to call back into the client when the
* transfer finishes. To do this we need the "request" that the client
* originally provided us. This function grabs the request that we stashed
* when we started the transfer.
*
* This only makes sense for ACTIVE_ONLY transfers since those are the only
* ones we track sending (the only ones we enable interrupts for and the only
* ones we call back to the client for).
*
* Return: The stashed request.
*/
static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv,
int tcs_id,
int *ch)
{
struct tcs_group *tcs;
int i;
for (i = 0; i < MAX_CHANNEL; i++) {
if (!drv->ch[i].initialized)
continue;
tcs = get_tcs_for_msg(drv, RPMH_ACTIVE_ONLY_STATE, i);
if (tcs->mask & BIT(tcs_id)) {
*ch = i;
return tcs->req[tcs_id - tcs->offset];
}
}
return NULL;
}
/**
* __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS
* @drv: The controller.
* @tcs_id: The global ID of this TCS.
* @trigger: If true then untrigger/retrigger. If false then just untrigger.
*
* In the normal case we only ever call with "trigger=true" to start a
* transfer. That will un-trigger/disable the TCS from the last transfer
* then trigger/enable for this transfer.
*
* If we borrowed a wake TCS for an active-only transfer we'll also call
* this function with "trigger=false" to just do the un-trigger/disable
* before using the TCS for wake purposes again.
*
* Note that the AP is only in charge of triggering active-only transfers.
* The AP never triggers sleep/wake values using this function.
*/
static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger)
{
u32 enable;
u32 reg = drv->regs[RSC_DRV_CONTROL];
/*
* HW req: Clear the DRV_CONTROL and enable TCS again
* While clearing ensure that the AMC mode trigger is cleared
* and then the mode enable is cleared.
*/
enable = read_tcs_reg(drv, reg, tcs_id);
enable &= ~TCS_AMC_MODE_TRIGGER;
write_tcs_reg_sync(drv, reg, tcs_id, enable);
enable &= ~TCS_AMC_MODE_ENABLE;
write_tcs_reg_sync(drv, reg, tcs_id, enable);
if (trigger) {
/* Enable the AMC mode on the TCS and then trigger the TCS */
enable = TCS_AMC_MODE_ENABLE;
write_tcs_reg_sync(drv, reg, tcs_id, enable);
enable |= TCS_AMC_MODE_TRIGGER;
write_tcs_reg(drv, reg, tcs_id, enable);
}
}
/**
* enable_tcs_irq() - Enable or disable interrupts on the given TCS.
* @drv: The controller.
* @tcs_id: The global ID of this TCS.
* @enable: If true then enable; if false then disable
*
* We only ever call this when we borrow a wake TCS for an active-only
* transfer. For active-only TCSes interrupts are always left enabled.
*/
static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable)
{
u32 data;
u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE];
data = readl_relaxed(drv->tcs_base + reg);
if (enable)
data |= BIT(tcs_id);
else
data &= ~BIT(tcs_id);
writel_relaxed(data, drv->tcs_base + reg);
}
/**
* tcs_tx_done() - TX Done interrupt handler.
* @irq: The IRQ number (ignored).
* @p: Pointer to "struct rsc_drv".
*
* Called for ACTIVE_ONLY transfers (those are the only ones we enable the
* IRQ for) when a transfer is done.
*
* Return: IRQ_HANDLED
*/
static irqreturn_t tcs_tx_done(int irq, void *p)
{
struct rsc_drv *drv = p;
int i, ch;
unsigned long irq_status;
const struct tcs_request *req;
irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]);
for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) {
req = get_req_from_tcs(drv, i, &ch);
if (WARN_ON(!req))
goto skip;
trace_rpmh_tx_done(drv, i, req);
#if IS_ENABLED(CONFIG_IPC_LOGGING)
ipc_log_string(drv->ipc_log_ctx, "IRQ response: m=%d", i);
#endif
/*
* If wake tcs was re-purposed for sending active
* votes, clear AMC trigger & enable modes and
* disable interrupt for this TCS
*/
if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs)
__tcs_set_trigger(drv, i, false);
skip:
/* Reclaim the TCS */
write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0);
write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], i, 0);
writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]);
spin_lock(&drv->lock);
clear_bit(i, drv->tcs_in_use);
/*
* Disable interrupt for WAKE TCS to avoid being
* spammed with interrupts coming when the solver
* sends its wake votes.
*/
if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs)
enable_tcs_irq(drv, i, false);
spin_unlock(&drv->lock);
wake_up(&drv->tcs_wait);
if (req)
rpmh_tx_done(req);
}
return IRQ_HANDLED;
}
/**
* __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger.
* @drv: The controller.
* @tcs_id: The global ID of this TCS.
* @cmd_id: The index within the TCS to start writing.
* @msg: The message we want to send, which will contain several addr/data
* pairs to program (but few enough that they all fit in one TCS).
*
* This is used for all types of transfers (active, sleep, and wake).
*/
static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id,
const struct tcs_request *msg)
{
u32 msgid;
u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE;
u32 cmd_enable = 0;
u32 cmd_complete;
struct tcs_cmd *cmd;
int i, j;
/* Convert all commands to RR when the request has wait_for_compl set */
cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0;
cmd_complete = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], tcs_id);
for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) {
cmd = &msg->cmds[i];
cmd_enable |= BIT(j);
cmd_complete |= cmd->wait << j;
msgid = cmd_msgid;
/*
* Additionally, if the cmd->wait is set, make the command
* response reqd even if the overall request was fire-n-forget.
*/
msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0;
write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid);
write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr);
write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data);
trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd);
#if IS_ENABLED(CONFIG_IPC_LOGGING)
ipc_log_string(drv->ipc_log_ctx,
"TCS write: m=%d n=%d msgid=%#x addr=%#x data=%#x wait=%d",
tcs_id, j, msgid, cmd->addr,
cmd->data, cmd->wait);
#endif
}
write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], tcs_id, cmd_complete);
cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id);
write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable);
}
/**
* check_for_req_inflight() - Look to see if conflicting cmds are in flight.
* @drv: The controller.
* @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers.
* @msg: The message we want to send, which will contain several addr/data
* pairs to program (but few enough that they all fit in one TCS).
*
* This will walk through the TCSes in the group and check if any of them
* appear to be sending to addresses referenced in the message. If it finds
* one it'll return -EBUSY.
*
* Only for use for active-only transfers.
*
* Must be called with the drv->lock held since that protects tcs_in_use.
*
* Return: 0 if nothing in flight or -EBUSY if we should try again later.
* The caller must re-enable interrupts between tries since that's
* the only way tcs_in_use will ever be updated and the only way
* RSC_DRV_CMD_ENABLE will ever be cleared.
*/
static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs,
const struct tcs_request *msg)
{
unsigned long curr_enabled;
u32 addr;
int j, k;
int i = tcs->offset;
unsigned long accl;
for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) {
curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i);
for_each_set_bit(j, &curr_enabled, tcs->ncpt) {
addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j);
for (k = 0; k < msg->num_cmds; k++) {
/*
* Each RPMh VREG accelerator resource has 3 or 4 contiguous 4-byte
* aligned addresses associated with it. Ignore the offset to check
* for in-flight VREG requests.
*/
accl = ACCL_TYPE(msg->cmds[k].addr);
if (accl == HW_ACCL_VREG &&
VREG_ADDR(addr) == VREG_ADDR(msg->cmds[k].addr))
return -EBUSY;
else if (addr == msg->cmds[k].addr)
return -EBUSY;
}
}
}
return 0;
}
/**
* find_free_tcs() - Find free tcs in the given tcs_group; only for active.
* @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if
* we borrowed it because there are zero active-only ones).
*
* Must be called with the drv->lock held since that protects tcs_in_use.
*
* Return: The first tcs that's free or -EBUSY if all in use.
*/
static int find_free_tcs(struct tcs_group *tcs)
{
const struct rsc_drv *drv = tcs->drv;
unsigned long i;
unsigned long max = tcs->offset + tcs->num_tcs;
int timeout = 100;
i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset);
if (i >= max)
return -EBUSY;
while (timeout) {
if (read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], i))
break;
timeout--;
udelay(1);
}
if (!timeout)
return -EBUSY;
return i;
}
/**
* claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active.
* @drv: The controller.
* @tcs: The tcs_group used for ACTIVE_ONLY transfers.
* @msg: The data to be sent.
*
* Claims a tcs in the given tcs_group while making sure that no existing cmd
* is in flight that would conflict with the one in @msg.
*
* Context: Must be called with the drv->lock held since that protects
* tcs_in_use.
*
* Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight
* or the tcs_group is full.
*/
static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs,
const struct tcs_request *msg)
{
int ret;
/*
* The h/w does not like if we send a request to the same address,
* when one is already in-flight or being processed.
*/
ret = check_for_req_inflight(drv, tcs, msg);
if (ret)
return ret;
return find_free_tcs(tcs);
}
/**
* rpmh_rsc_send_data() - Write / trigger active-only message.
* @drv: The controller.
* @msg: The data to be sent.
* @ch: Channel number
*
* NOTES:
* - This is only used for "ACTIVE_ONLY" since the limitations of this
* function don't make sense for sleep/wake cases.
* - To do the transfer, we will grab a whole TCS for ourselves--we don't
* try to share. If there are none available we'll wait indefinitely
* for a free one.
* - This function will not wait for the commands to be finished, only for
* data to be programmed into the RPMh. See rpmh_tx_done() which will
* be called when the transfer is fully complete.
* - This function must be called with interrupts enabled. If the hardware
* is busy doing someone else's transfer we need that transfer to fully
* finish so that we can have the hardware, and to fully finish it needs
* the interrupt handler to run. If the interrupts is set to run on the
* active CPU this can never happen if interrupts are disabled.
*
* Return: 0 on success, -EINVAL on error.
*/
int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch)
{
struct tcs_group *tcs;
int tcs_id;
unsigned long flags;
tcs = get_tcs_for_msg(drv, msg->state, ch);
if (IS_ERR(tcs))
return PTR_ERR(tcs);
spin_lock_irqsave(&drv->lock, flags);
/* Controller is busy in 'solver' mode */
if (drv->in_solver_mode) {
spin_unlock_irqrestore(&drv->lock, flags);
return -EBUSY;
}
/* Wait forever for a free tcs. It better be there eventually! */
wait_event_lock_irq(drv->tcs_wait,
(tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0,
drv->lock);
tcs->req[tcs_id - tcs->offset] = msg;
set_bit(tcs_id, drv->tcs_in_use);
/*
* Clear previously programmed ACTIVE/WAKE commands in selected
* repurposed TCS to avoid triggering them. tcs->slots will be
* cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate()
*/
write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0);
write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], tcs_id, 0);
if (msg->wait_for_compl || (msg->state == RPMH_ACTIVE_ONLY_STATE &&
tcs->type != ACTIVE_TCS))
enable_tcs_irq(drv, tcs_id, true);
else
enable_tcs_irq(drv, tcs_id, false);
/*
* These two can be done after the lock is released because:
* - We marked "tcs_in_use" under lock.
* - Once "tcs_in_use" has been marked nobody else could be writing
* to these registers until the interrupt goes off.
* - The interrupt can't go off until we trigger w/ the last line
* of __tcs_set_trigger() below.
*/
__tcs_buffer_write(drv, tcs_id, 0, msg);
__tcs_set_trigger(drv, tcs_id, true);
#if IS_ENABLED(CONFIG_IPC_LOGGING)
ipc_log_string(drv->ipc_log_ctx, "TCS trigger: m=%d wait_for_compl=%u",
tcs_id, msg->wait_for_compl);
#endif
if (!msg->wait_for_compl)
clear_bit(tcs_id, drv->tcs_in_use);
spin_unlock_irqrestore(&drv->lock, flags);
if (!msg->wait_for_compl)
wake_up(&drv->tcs_wait);
return 0;
}
/**
* find_slots() - Find a place to write the given message.
* @tcs: The tcs group to search.
* @msg: The message we want to find room for.
* @tcs_id: If we return 0 from the function, we return the global ID of the
* TCS to write to here.
* @cmd_id: If we return 0 from the function, we return the index of
* the command array of the returned TCS where the client should
* start writing the message.
*
* Only for use on sleep/wake TCSes since those are the only ones we maintain
* tcs->slots for.
*
* Return: -ENOMEM if there was no room, else 0.
*/
static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg,
int *tcs_id, int *cmd_id)
{
int slot, offset;
int i = 0;
/* Do over, until we can fit the full payload in a single TCS */
do {
slot = bitmap_find_next_zero_area(tcs->slots,
tcs->ncpt * tcs->num_tcs,
i, msg->num_cmds, 0);
if (slot >= tcs->num_tcs * tcs->ncpt)
return -ENOMEM;
i += tcs->ncpt;
} while (slot + msg->num_cmds - 1 >= i);
bitmap_set(tcs->slots, slot, msg->num_cmds);
offset = slot / tcs->ncpt;
*tcs_id = offset + tcs->offset;
*cmd_id = slot % tcs->ncpt;
return 0;
}
/**
* rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger.
* @drv: The controller.
* @msg: The data to be written to the controller.
* @ch: Channel number
*
* This should only be called for sleep/wake state, never active-only
* state.
*
* The caller must ensure that no other RPMH actions are happening and the
* controller is idle when this function is called since it runs lockless.
*
* Return: 0 if no error; else -error.
*/
int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch)
{
struct tcs_group *tcs;
int tcs_id = 0, cmd_id = 0;
int ret;
if (!msg->num_cmds) {
#if IS_ENABLED(CONFIG_IPC_LOGGING)
ipc_log_string(drv->ipc_log_ctx, "Empty num_cmds, returning");
#endif
return 0;
}
tcs = get_tcs_for_msg(drv, msg->state, ch);
if (IS_ERR(tcs))
return PTR_ERR(tcs);
/* find the TCS id and the command in the TCS to write to */
ret = find_slots(tcs, msg, &tcs_id, &cmd_id);
if (!ret)
__tcs_buffer_write(drv, tcs_id, cmd_id, msg);
return ret;
}
static struct tcs_group *get_tcs_from_index(struct rsc_drv *drv, int tcs_id)
{
unsigned int i, j;
for (i = 0; i < TCS_TYPE_NR; i++) {
for (j = 0; j < MAX_CHANNEL; j++) {
if (!drv->ch[j].initialized)
continue;
if (drv->ch[j].tcs[i].mask & BIT(tcs_id))
return &drv->ch[j].tcs[i];
}
}
return NULL;
}
static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl,
bool *aoss_irq_sts)
{
int ch = 0;
struct tcs_group *tcs_grp = get_tcs_from_index(drv, tcs_id);
const struct tcs_request *req = get_req_from_tcs(drv, tcs_id, &ch);
unsigned long cmds_enabled;
u32 addr, data, msgid, sts, irq_sts;
bool in_use = test_bit(tcs_id, drv->tcs_in_use);
int i;
sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id);
cmds_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id);
if (!cmds_enabled || !tcs_grp)
return;
if (!req)
goto print_tcs_data;
data = read_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id);
irq_sts = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]);
pr_warn("Request: tcs-in-use:%s active_tcs=%s(%d) state=%d wait_for_compl=%u]\n",
(in_use ? "YES" : "NO"),
((tcs_grp->type == ACTIVE_TCS) ? "YES" : "NO"),
tcs_grp->type, req->state, req->wait_for_compl);
pr_warn("TCS=%d [ctrlr-sts:%s amc-mode:0x%x irq-sts:%s]\n",
tcs_id, sts ? "IDLE" : "BUSY", data,
(irq_sts & BIT(tcs_id)) ? "COMPLETED" : "PENDING");
*aoss_irq_sts = (irq_sts & BIT(tcs_id)) ? true : false;
print_tcs_data:
for_each_set_bit(i, &cmds_enabled, tcs_grp->ncpt) {
addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, i);
data = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i);
msgid = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, i);
sts = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_STATUS], tcs_id, i);
pr_warn("\tCMD=%d [addr=0x%x data=0x%x hdr=0x%x sts=0x%x enabled=1]\n",
i, addr, data, msgid, sts);
if (!(sts & CMD_STATUS_ISSUED))
continue;
if (!(sts & CMD_STATUS_COMPL))
*accl |= BIT(ACCL_TYPE(addr));
}
}
void rpmh_rsc_debug_channel_busy(struct rsc_drv *drv)
{
u32 event_sts, ctrl_sts;
u32 chn_update, chn_busy, chn_en;
u32 seq_busy, seq_pc;
pr_err("RSC:%s\n", drv->name);
event_sts = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_TCS_COMPLETE]);
ctrl_sts = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_TCS_TRIGGER]);
chn_update = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_UPDATE]);
chn_busy = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_BUSY]);
chn_en = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_EN]);
seq_busy = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_SEQ_BUSY]);
seq_pc = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_SEQ_PC]);
pr_err("event sts: 0x%x ctrl_sts: 0x%x\n", event_sts, ctrl_sts);
pr_err("chn_update: 0x%x chn_busy: 0x%x chn_en: 0x%x\n", chn_update, chn_busy, chn_en);
pr_err("seq_busy: 0x%x seq_pc: 0x%x\n", seq_busy, seq_pc);
crm_dump_regs("cam_crm");
crm_dump_drv_regs("cam_crm", drv->id);
}
void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl)
{
struct irq_data *rsc_irq_data = irq_get_irq_data(drv->irq);
bool gic_irq_sts, aoss_irq_sts = false;
int i;
int busy = 0;
unsigned long accl = 0;
char str[20] = "";
pr_warn("RSC:%s\n", drv->name);
for (i = 0; i < drv->num_tcs; i++) {
if (!test_bit(i, drv->tcs_in_use))
continue;
busy++;
print_tcs_info(drv, i, &accl, &aoss_irq_sts);
}
if (!rsc_irq_data) {
pr_err("No IRQ data for RSC:%s\n", drv->name);
return;
}
irq_get_irqchip_state(drv->irq, IRQCHIP_STATE_PENDING, &gic_irq_sts);
pr_warn("HW IRQ %lu is %s at GIC\n", rsc_irq_data->hwirq,
gic_irq_sts ? "PENDING" : "NOT PENDING");
pr_warn("Completion is %s to finish\n",
completion_done(compl) ? "PENDING" : "NOT PENDING");
for_each_set_bit(i, &accl, ARRAY_SIZE(accl_str)) {
strlcat(str, accl_str[i], sizeof(str));
strlcat(str, " ", sizeof(str));
}
if ((busy && !gic_irq_sts) || !aoss_irq_sts)
pr_warn("ERROR:Accelerator(s) { %s } at AOSS did not respond\n",
str);
else if (gic_irq_sts)
pr_warn("ERROR:Possible lockup in Linux\n");
/* Show fast path status, if the TCS is busy */
for (i = 0; i < MAX_CHANNEL; i++) {
if (!drv->ch[i].initialized)
continue;
/* Show fast path status, if the TCS is busy */
if (drv->ch[i].tcs[FAST_PATH_TCS].num_tcs) {
int tcs_id = drv->ch[i].tcs[FAST_PATH_TCS].offset;
bool sts = read_tcs_reg(drv,
drv->regs[RSC_DRV_STATUS],
tcs_id);
if (!sts) {
pr_err("Fast-path TCS information:\n");
print_tcs_info(drv, tcs_id, &accl, &aoss_irq_sts);
}
}
}
/*
* The TCS(s) are busy waiting, we have no way to recover from this.
* If this debug function is called, we assume it's because timeout
* has happened.
* Crash and report.
*/
BUG_ON(busy);
}
/**
* rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy.
* @drv: The controller
*
* Checks if any of the AMCs are busy in handling ACTIVE sets.
* This is called from the last cpu powering down before flushing
* SLEEP and WAKE sets. If AMCs are busy, controller can not enter
* power collapse, so deny from the last cpu's pm notification.
*
* Context: Must be called with the drv->lock held.
*
* Return:
* * False - AMCs are idle
* * True - AMCs are busy
*/
static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv)
{
int i;
struct tcs_group *tcs;
unsigned long set;
unsigned long max;
for (i = 0; i < MAX_CHANNEL; i++) {
if (!drv->ch[i].initialized)
continue;
tcs = &drv->ch[i].tcs[ACTIVE_TCS];
/*
* If we made an active request on a RSC that does not have a
* dedicated TCS for active state use, then re-purposed wake TCSes
* should be checked for not busy, because we used wake TCSes for
* active requests in this case.
*/
if (!tcs->num_tcs)
tcs = &drv->ch[i].tcs[WAKE_TCS];
max = tcs->offset + tcs->num_tcs;
set = find_next_bit(drv->tcs_in_use, max, tcs->offset);
if (set < max)
return true;
/* Check if there is pending fastpath transaction */
tcs = &drv->ch[i].tcs[FAST_PATH_TCS];
if (tcs->num_tcs &&
!read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs->offset))
return true;
}
return false;
}
/**
* rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS.
* @drv: The controller
*
* Writes maximum wakeup cycles when called from suspend.
* Writes earliest hrtimer wakeup when called from idle.
*/
void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv)
{
ktime_t now, wakeup;
u64 wakeup_us, wakeup_cycles = ~0;
u32 lo, hi;
if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call)
return;
/* Set highest time when system (timekeeping) is suspended */
if (system_state == SYSTEM_SUSPEND)
goto exit;
/* Find the earliest hrtimer wakeup from online cpus */
wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev);
/* Find the relative wakeup in kernel time scale */
now = ktime_get();
wakeup = ktime_sub(wakeup, now);
wakeup_us = ktime_to_us(wakeup);
/* Convert the wakeup to arch timer scale */
wakeup_cycles = USECS_TO_CYCLES(wakeup_us);
wakeup_cycles += arch_timer_read_counter();
exit:
lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK;
hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE;
hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK;
hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID;
writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO);
writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI);
}
/**
* rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy.
* @nfb: Pointer to the notifier block in struct rsc_drv.
* @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT.
* @v: Unused
*
* This function is given to cpu_pm_register_notifier so we can be informed
* about when CPUs go down. When all CPUs go down we know no more active
* transfers will be started so we write sleep/wake sets. This function gets
* called from cpuidle code paths and also at system suspend time.
*
* If its last CPU going down and AMCs are not busy then writes cached sleep
* and wake messages to TCSes. The firmware then takes care of triggering
* them when entering deepest low power modes.
*
* Return: See cpu_pm_register_notifier()
*/
static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb,
unsigned long action, void *v)
{
struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm);
int ret = NOTIFY_OK;
int cpus_in_pm, ch;
switch (action) {
case CPU_PM_ENTER:
cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm);
/*
* NOTE: comments for num_online_cpus() point out that it's
* only a snapshot so we need to be careful. It should be OK
* for us to use, though. It's important for us not to miss
* if we're the last CPU going down so it would only be a
* problem if a CPU went offline right after we did the check
* AND that CPU was not idle AND that CPU was the last non-idle
* CPU. That can't happen. CPUs would have to come out of idle
* before the CPU could go offline.
*/
if (cpus_in_pm < num_online_cpus())
return NOTIFY_OK;
break;
case CPU_PM_ENTER_FAILED:
case CPU_PM_EXIT:
atomic_dec(&drv->cpus_in_pm);
return NOTIFY_OK;
default:
return NOTIFY_DONE;
}
/*
* It's likely we're on the last CPU. Grab the drv->lock and write
* out the sleep/wake commands to RPMH hardware. Grabbing the lock
* means that if we race with another CPU coming up we are still
* guaranteed to be safe. If another CPU came up just after we checked
* and has grabbed the lock or started an active transfer then we'll
* notice we're busy and abort. If another CPU comes up after we start
* flushing it will be blocked from starting an active transfer until
* we're done flushing. If another CPU starts an active transfer after
* we release the lock we're still OK because we're no longer the last
* CPU.
*/
if (spin_trylock(&drv->lock)) {
ch = rpmh_rsc_get_channel(drv);
if (ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client, ch))
ret = NOTIFY_BAD;
spin_unlock(&drv->lock);
} else {
/* Another CPU must be up */
return NOTIFY_OK;
}
if (ret == NOTIFY_BAD) {
/* Double-check if we're here because someone else is up */
if (cpus_in_pm < num_online_cpus())
ret = NOTIFY_OK;
else
/* We won't be called w/ CPU_PM_ENTER_FAILED */
atomic_dec(&drv->cpus_in_pm);
}
return ret;
}
/**
* rpmh_rsc_mode_solver_set() - Enable/disable solver mode.
* @drv: The controller.
* @enable: Boolean state to be set - true/false
*
* Return:
* * 0 - success
* * -EBUSY - AMCs are busy
*/
int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable)
{
int ret = -EBUSY;
if (spin_trylock(&drv->lock)) {
if (!enable || !rpmh_rsc_ctrlr_is_busy(drv)) {
drv->in_solver_mode = enable;
trace_rpmh_solver_set(drv, enable);
#if IS_ENABLED(CONFIG_IPC_LOGGING)
ipc_log_string(drv->ipc_log_ctx,
"solver mode set: %d", enable);
#endif
ret = 0;
}
spin_unlock(&drv->lock);
}
return ret;
}
int rpmh_rsc_is_tcs_completed(struct rsc_drv *drv, int ch)
{
u32 sts;
int retry = 10, ret = 0;
do {
sts = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_TCS_COMPLETE]);
if (ch == 0)
sts &= CH0_WAKE_TCS_STATUS;
else
sts &= CH1_WAKE_TCS_STATUS;
retry--;
/*
* Wait till all the WAKE votes of the new channel are
* applied during channel switch.
* Maximum delay of 100 usec.
*/
if (!sts)
udelay(10);
} while (!sts && retry);
if (!retry) {
ret = -EBUSY;
goto exit;
}
writel_relaxed(CH_CLEAR_STATUS,
drv->base + drv->regs[RSC_DRV_CHN_TCS_COMPLETE]);
exit:
trace_rpmh_switch_channel(drv, ch, ret);
#if IS_ENABLED(CONFIG_IPC_LOGGING)
ipc_log_string(drv->ipc_log_ctx, "channel switched to: %d ret: %d", ch, ret);
#endif
return 0;
}
/**
* rpmh_rsc_switch_channel() - Switch to the channel
* @drv: The controller.
* @ch: The channel number to switch to.
*
* NOTE: Caller should ensure serialization before making this call.
* Return:
* * 0 - success
* * -Error - Error code
*/
int rpmh_rsc_switch_channel(struct rsc_drv *drv, int ch)
{
writel_relaxed(BIT(ch), drv->base + drv->regs[RSC_DRV_CHN_UPDATE]);
return rpmh_rsc_is_tcs_completed(drv, ch);
}
/**
* rpmh_rsc_drv_enable() - Enable the DRV and trigger Wake vote
* @drv: The controller.
*
* NOTE: Caller should ensure serialization before making this call.
* Return:
* * 0 - success
* * -Error - Error code
*/
int rpmh_rsc_drv_enable(struct rsc_drv *drv, bool enable)
{
int ret = 0, ch;
u32 chn_en;
spin_lock(&drv->lock);
chn_en = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_EN]);
if (chn_en == enable) {
ret = -EINVAL;
goto exit;
}
if (enable) {
/* Start with channel 0 */
ch = 0;
ret = rpmh_flush(&drv->client, ch);
if (ret)
goto exit;
writel_relaxed(enable, drv->base + drv->regs[RSC_DRV_CHN_EN]);
ret = rpmh_rsc_switch_channel(drv, ch);
if (ret)
goto exit;
} else {
/* Select unused channel */
ch = rpmh_rsc_get_channel(drv);
if (ch < 0)
goto exit;
ret = rpmh_flush(&drv->client, ch);
if (ret)
goto exit;
ret = rpmh_rsc_switch_channel(drv, ch);
if (ret)
goto exit;
writel_relaxed(0, drv->base + drv->regs[RSC_DRV_CHN_UPDATE]);
writel_relaxed(enable, drv->base + drv->regs[RSC_DRV_CHN_EN]);
}
exit:
spin_unlock(&drv->lock);
trace_rpmh_drv_enable(drv, enable, ret);
#if IS_ENABLED(CONFIG_IPC_LOGGING)
ipc_log_string(drv->ipc_log_ctx, "drv enable: %d ret: %d", enable, ret);
#endif
return ret;
}
/**
* rpmh_rsc_init_fast_path() - Initialize the fast-path TCS contents
* @drv: The controller.
* @msg: The TCS request to populate.
* @ch: Channel number
*
* Return:
* * 0 - success
* * -ENODEV - no fast-path TCS available
*/
int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, int ch)
{
int tcs_id;
if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs)
return -ENODEV;
tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset;
/* We won't use the AMC IRQ to confirm if the TCS is free */
enable_tcs_irq(drv, tcs_id, false);
__tcs_buffer_write(drv, tcs_id, 0, msg);
return 0;
}
/**
* rpmh_rsc_update_fast_path() - Update the fast-path TCS data and trigger
* @drv: The controller.
* @msg: The TCS request data to be updated.
* @mask: The update mask for elements in @msg to be sent
* @ch: Channel number
*
* NOTE: Caller should ensure serialization before making this call.
* Return:
* * 0 - success
* * -ENODEV - no fast-path TCS available
*/
int rpmh_rsc_update_fast_path(struct rsc_drv *drv,
const struct tcs_request *msg,
u32 mask, int ch)
{
int i;
u32 sts;
int tcs_id;
struct tcs_cmd *cmd;
int retry = 5;
if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs)
return -ENODEV;
tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset;
/* Ensure the TCS is free before writing to the TCS */
do {
sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id);
if (!sts) {
retry--;
/* Report and bail, if it took too many attempts */
if (!retry) {
pr_err("Fast-path TCS is too busy\n");
return -EBUSY;
}
udelay(1);
}
} while (!sts);
/*
* We only update the data, everything else remains the same.
* The number of commands and the addresses do not change with
* updates.
*/
for (i = 0; i < msg->num_cmds; i++) {
if (!(mask & BIT(i)))
continue;
cmd = &msg->cmds[i];
write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i, cmd->data);
}
/* Trigger the TCS to send the request */
__tcs_set_trigger(drv, tcs_id, true);
return 0;
}
static int rpmh_rsc_poweroff_noirq(struct device *dev)
{
return 0;
}
static void rpmh_rsc_tcs_irq_enable(struct rsc_drv *drv)
{
u32 tcs_mask;
int ch;
for (ch = 0; ch < MAX_CHANNEL; ch++) {
if (!drv->ch[ch].initialized)
continue;
tcs_mask = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);
tcs_mask |= drv->ch[ch].tcs[ACTIVE_TCS].mask;
writel_relaxed(tcs_mask, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);
}
}
static int rpmh_rsc_restore_noirq(struct device *dev)
{
struct rsc_drv_top *rsc_top = dev_get_drvdata(dev);
int i;
for (i = 0; i < rsc_top->drv_count; i++) {
if (rsc_top->drv[i].initialized)
rpmh_rsc_tcs_irq_enable(&rsc_top->drv[i]);
}
return 0;
}
static struct rsc_drv_top *rpmh_rsc_get_top_device(const char *name)
{
struct rsc_drv_top *rsc_top;
bool rsc_dev_present = false;
list_for_each_entry(rsc_top, &rpmh_rsc_dev_list, list) {
if (!strcmp(name, rsc_top->name)) {
rsc_dev_present = true;
break;
}
}
if (!rsc_dev_present)
return ERR_PTR(-ENODEV);
return rsc_top;
}
const struct device *rpmh_rsc_get_device(const char *name, u32 drv_id)
{
struct rsc_drv_top *rsc_top = rpmh_rsc_get_top_device(name);
int i;
if (IS_ERR(rsc_top) || strcmp(name, "cam_rsc"))
return ERR_PTR(-ENODEV);
for (i = 0; i < rsc_top->drv_count; i++) {
if (i == drv_id && rsc_top->drv[i].initialized)
return rsc_top->drv[i].dev;
}
return ERR_PTR(-ENODEV);
}
static int rpmh_probe_channel_tcs_config(struct device_node *np,
struct rsc_drv *drv,
u32 max_tcs, u32 ncpt, int ch)
{
struct tcs_type_config {
u32 type;
u32 n;
} tcs_cfg[TCS_TYPE_NR] = { { 0 } };
struct tcs_group *tcs;
struct drv_channel *channel = &drv->ch[ch];
int i, ret, n, st = 0;
u32 tcs_mask;
n = of_property_count_u32_elems(np, "qcom,tcs-config");
if (n != 2 * TCS_TYPE_NR)
return -EINVAL;
for (i = 0; i < TCS_TYPE_NR; i++) {
ret = of_property_read_u32_index(np, "qcom,tcs-config",
i * 2, &tcs_cfg[i].type);
if (ret)
return ret;
if (tcs_cfg[i].type >= TCS_TYPE_NR)
return -EINVAL;
ret = of_property_read_u32_index(np, "qcom,tcs-config",
i * 2 + 1, &tcs_cfg[i].n);
if (ret)
return ret;
if (tcs_cfg[i].n > MAX_TCS_PER_TYPE)
return -EINVAL;
}
for (i = 0; i < TCS_TYPE_NR; i++) {
tcs = &channel->tcs[tcs_cfg[i].type];
if (tcs->drv)
return -EINVAL;
tcs->drv = drv;
tcs->type = tcs_cfg[i].type;
tcs->num_tcs = tcs_cfg[i].n;
tcs->ncpt = ncpt;
if (!tcs->num_tcs || tcs->type == CONTROL_TCS)
continue;
if (st + tcs->num_tcs > max_tcs ||
st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask))
return -EINVAL;
tcs->mask = ((1 << tcs->num_tcs) - 1) << (st + drv->num_tcs);
tcs->offset = st + drv->num_tcs;
st += tcs->num_tcs;
}
/* Enable the active TCS to send requests immediately */
tcs_mask = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);
tcs_mask |= drv->ch[ch].tcs[ACTIVE_TCS].mask;
writel_relaxed(tcs_mask, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]);
channel->drv = drv;
channel->initialized = true;
drv->num_tcs += st;
return 0;
}
/**
* rpmh_rsc_pd_callback() - Check if any of the AMCs are busy.
* @nfb: Pointer to the genpd notifier block in struct rsc_drv.
* @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON.
* @v: Unused
*
* This function is given to dev_pm_genpd_add_notifier() so we can be informed
* about when cluster-pd is going down. When cluster go down we know no more active
* transfers will be started so we write sleep/wake sets. This function gets
* called from cpuidle code paths and also at system suspend time.
*
* If AMCs are not busy then writes cached sleep and wake messages to TCSes.
* The firmware then takes care of triggering them when entering deepest low power modes.
*
* Return:
* * NOTIFY_OK - success
* * NOTIFY_BAD - failure
*/
static int rpmh_rsc_pd_callback(struct notifier_block *nfb,
unsigned long action, void *v)
{
struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb);
int ch;
ch = rpmh_rsc_get_channel(drv);
/* We don't need to lock as genpd on/off are serialized */
if ((action == GENPD_NOTIFY_PRE_OFF) &&
(ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || _rpmh_flush(&drv->client, ch)))
return NOTIFY_BAD;
return NOTIFY_OK;
}
static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev)
{
int ret;
pm_runtime_enable(dev);
drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback;
ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb);
if (ret)
pm_runtime_disable(dev);
return ret;
}
static int rpmh_probe_tcs_config(struct rsc_drv *drv)
{
struct device_node *cn, *np = drv->dev->of_node;
int ch = 0, ret;
u32 offset, config;
u32 max_tcs, ncpt;
ret = of_property_read_u32(np, "qcom,tcs-offset", &offset);
if (ret)
return ret;
drv->tcs_base = drv->base + offset;
ret = of_property_read_u32(np, "qcom,tcs-distance", &drv->tcs_distance);
if (ret)
drv->tcs_distance = 0;
config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]);
max_tcs = config;
max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id);
max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id);
ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT);
ncpt = ncpt >> DRV_NCPT_SHIFT;
for_each_child_of_node(np, cn) {
if (!of_node_name_eq(cn, "channel"))
continue;
ret = rpmh_probe_channel_tcs_config(cn, drv, max_tcs, ncpt, ch);
if (ret)
return ret;
ch++;
}
drv->num_channels = ch;
return 0;
}
static int rpmh_rsc_probe(struct platform_device *pdev)
{
struct device_node *np, *dn = pdev->dev.of_node;
struct rsc_drv *drv;
struct rsc_drv_top *rsc_top;
int ret, irq;
u32 rsc_id, major_ver, minor_ver, solver_config;
int i, j, drv_count;
const char *name;
/*
* Even though RPMh doesn't directly use cmd-db, all of its children
* do. To avoid adding this check to our children we'll do it now.
*/
ret = cmd_db_ready();
if (ret) {
if (ret != -EPROBE_DEFER)
dev_err(&pdev->dev, "Command DB not available (%d)\n",
ret);
return ret;
}
rpmh_standalone = cmd_db_is_standalone();
if (rpmh_standalone)
dev_info(&pdev->dev, "RPMH is running in standalone mode.\n");
rsc_top = devm_kzalloc(&pdev->dev, sizeof(*rsc_top), GFP_KERNEL);
if (!rsc_top)
return -ENOMEM;
ret = of_property_read_u32(dn, "qcom,drv-count", &drv_count);
if (ret)
return ret;
drv = devm_kcalloc(&pdev->dev, drv_count, sizeof(*drv), GFP_KERNEL);
if (!drv)
return -ENOMEM;
name = of_get_property(dn, "label", NULL);
if (!name)
name = dev_name(&pdev->dev);
rsc_top->drv_count = drv_count;
rsc_top->drv = drv;
rsc_top->dev = &pdev->dev;
scnprintf(rsc_top->name, sizeof(rsc_top->name), "%s", name);
for_each_child_of_node(dn, np) {
struct device *drv_dev;
if (!of_node_name_eq(np, "drv"))
continue;
ret = of_property_read_u32(np, "qcom,drv-id", &i);
if (ret)
return ret;
scnprintf(drv[i].name, sizeof(drv[i].name), "%s-drv-%d", name, i);
drv[i].base = devm_platform_ioremap_resource(pdev, i);
if (IS_ERR(drv[i].base))
return PTR_ERR(drv[i].base);
drv_dev = kzalloc(sizeof(*drv_dev), GFP_KERNEL);
if (!drv_dev)
return -ENOMEM;
drv[i].id = i;
drv[i].pdev = pdev;
drv[i].dev = drv_dev;
drv_dev->parent = &pdev->dev;
drv_dev->of_node = np;
dev_set_name(drv_dev, "%s:%pOFn%d", dev_name(drv_dev->parent), np, i);
ret = device_register(drv_dev);
if (ret)
return ret;
rsc_id = readl_relaxed(drv[i].base + RSC_DRV_ID);
major_ver = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT);
major_ver >>= MAJOR_VER_SHIFT;
minor_ver = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT);
minor_ver >>= MINOR_VER_SHIFT;
if (major_ver >= 3)
drv[i].regs = rpmh_rsc_reg_offset_ver_3_0;
else
drv[i].regs = rpmh_rsc_reg_offset_ver_2_7;
ret = rpmh_probe_tcs_config(&drv[i]);
if (ret)
return ret;
dev_set_drvdata(drv_dev, &drv[i]);
drv[i].initialized = true;
}
for (i = 0; i < drv_count; i++) {
if (!drv[i].initialized)
continue;
/*
* CPU PM notification are not required for controllers that support
* 'HW solver' mode where they can be in autonomous mode executing low
* power mode to power down.
*/
solver_config = readl_relaxed(drv[i].base +
drv[i].regs[DRV_SOLVER_CONFIG]);
solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT;
solver_config = solver_config >> DRV_HW_SOLVER_SHIFT;
spin_lock_init(&drv[i].lock);
spin_lock_init(&drv[i].client.cache_lock);
if (of_find_property(dn, "power-domains", NULL) && pdev->dev.pm_domain) {
ret = rpmh_rsc_pd_attach(&drv[i], &pdev->dev);
if (ret)
return ret;
} else if (!solver_config &&
!of_find_property(dn, "qcom,hw-channel", NULL)) {
drv[i].rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback;
cpu_pm_register_notifier(&drv[i].rsc_pm);
} else if (solver_config) {
drv[i].client.flags = SOLVER_PRESENT;
} else {
/*
* The requets for HW channel TCSes has to be either
* RPMH_SLEEP_STATE or RPMH_WAKE_ONLY_STATE.
*
* Assume 'solver' state which does nothing but to disallow
* RPMH_ACTIVE_ONLY_STATE requests.
*/
drv[i].client.flags = SOLVER_PRESENT | HW_CHANNEL_PRESENT;
drv[i].client.in_solver_mode = true;
drv[i].in_solver_mode = true;
drv[i].regs = rpmh_rsc_reg_offset_ver_3_0_hw_channel;
}
init_waitqueue_head(&drv[i].tcs_wait);
bitmap_zero(drv[i].tcs_in_use, MAX_TCS_NR);
drv[i].client.non_batch_cache = devm_kcalloc(&pdev->dev, CMD_DB_MAX_RESOURCES,
sizeof(struct cache_req), GFP_KERNEL);
if (!drv[i].client.non_batch_cache)
return -ENOMEM;
for (j = 0; j < CMD_DB_MAX_RESOURCES; j++)
INIT_LIST_HEAD(&drv[i].client.non_batch_cache[j].list);
irq = platform_get_irq(pdev, drv[i].id);
if (irq < 0)
return irq;
drv[i].irq = irq;
ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done,
IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND,
drv[i].name, &drv[i]);
if (ret)
return ret;
#if IS_ENABLED(CONFIG_IPC_LOGGING)
drv[i].ipc_log_ctx = ipc_log_context_create(
RSC_DRV_IPC_LOG_SIZE,
drv[i].name, 0);
#endif
if (__rsc_count < MAX_RSC_COUNT)
__rsc_drv[__rsc_count++] = &drv[i];
ret = devm_of_platform_populate(drv[i].dev);
if (ret)
return ret;
}
INIT_LIST_HEAD(&rsc_top->list);
list_add_tail(&rsc_top->list, &rpmh_rsc_dev_list);
dev_set_drvdata(&pdev->dev, rsc_top);
return devm_of_platform_populate(&pdev->dev);
}
static const struct dev_pm_ops rpmh_rsc_dev_pm_ops = {
.poweroff_noirq = rpmh_rsc_poweroff_noirq,
.restore_noirq = rpmh_rsc_restore_noirq,
};
static const struct of_device_id rpmh_drv_match[] = {
{ .compatible = "qcom,rpmh-rsc", },
{ }
};
MODULE_DEVICE_TABLE(of, rpmh_drv_match);
static struct platform_driver rpmh_driver = {
.probe = rpmh_rsc_probe,
.driver = {
.name = "rpmh",
.of_match_table = rpmh_drv_match,
.pm = &rpmh_rsc_dev_pm_ops,
.suppress_bind_attrs = true,
},
};
static int __init rpmh_driver_init(void)
{
return platform_driver_register(&rpmh_driver);
}
arch_initcall(rpmh_driver_init);
MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver");
MODULE_LICENSE("GPL v2");
Reference in New Issue View Git Blame Copy Permalink