// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2016-2018, The Linux Foundation. All rights reserved. * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved. */ #define pr_fmt(fmt) "%s " fmt, KBUILD_MODNAME #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "rpmh-internal.h" #define CREATE_TRACE_POINTS #include "trace-rpmh.h" #if IS_ENABLED(CONFIG_IPC_LOGGING) #include #endif #define RSC_DRV_IPC_LOG_SIZE 2 /* DRV ID Register */ #define RSC_DRV_ID 0 #define MAJOR_VER_MASK 0xFF #define MAJOR_VER_SHIFT 16 #define MINOR_VER_MASK 0xFF #define MINOR_VER_SHIFT 8 enum { RSC_DRV_TCS_OFFSET, RSC_DRV_CMD_OFFSET, DRV_SOLVER_CONFIG, DRV_PRNT_CHLD_CONFIG, RSC_DRV_IRQ_ENABLE, RSC_DRV_IRQ_STATUS, RSC_DRV_IRQ_CLEAR, RSC_DRV_CMD_WAIT_FOR_CMPL, RSC_DRV_CONTROL, RSC_DRV_STATUS, RSC_DRV_CMD_ENABLE, RSC_DRV_CMD_MSGID, RSC_DRV_CMD_ADDR, RSC_DRV_CMD_DATA, RSC_DRV_CMD_STATUS, RSC_DRV_CMD_RESP_DATA, /* DRV channel Registers */ RSC_DRV_CHN_TCS_TRIGGER, RSC_DRV_CHN_TCS_COMPLETE, RSC_DRV_CHN_SEQ_BUSY, RSC_DRV_CHN_SEQ_PC, RSC_DRV_CHN_UPDATE, RSC_DRV_CHN_BUSY, RSC_DRV_CHN_EN, }; /* DRV HW Solver Configuration Information Register */ #define DRV_HW_SOLVER_MASK 1 #define DRV_HW_SOLVER_SHIFT 24 /* DRV TCS Configuration Information Register */ #define DRV_NUM_TCS_MASK 0x3F #define DRV_NUM_TCS_SHIFT 6 #define DRV_NCPT_MASK 0x1F #define DRV_NCPT_SHIFT 27 /* Offsets for CONTROL TCS Registers */ #define RSC_DRV_CTL_TCS_DATA_HI 0x38 #define RSC_DRV_CTL_TCS_DATA_HI_MASK 0xFFFFFF #define RSC_DRV_CTL_TCS_DATA_HI_VALID BIT(31) #define RSC_DRV_CTL_TCS_DATA_LO 0x40 #define RSC_DRV_CTL_TCS_DATA_LO_MASK 0xFFFFFFFF #define RSC_DRV_CTL_TCS_DATA_SIZE 32 #define TCS_AMC_MODE_ENABLE BIT(16) #define TCS_AMC_MODE_TRIGGER BIT(24) /* TCS CMD register bit mask */ #define CMD_MSGID_LEN 8 #define CMD_MSGID_RESP_REQ BIT(8) #define CMD_MSGID_WRITE BIT(16) #define CMD_STATUS_ISSUED BIT(8) #define CMD_STATUS_COMPL BIT(16) /* Offsets for DRV channel status register */ #define CH0_CHN_BUSY BIT(0) #define CH1_CHN_BUSY BIT(1) #define CH0_WAKE_TCS_STATUS BIT(0) #define CH0_SLEEP_TCS_STATUS BIT(1) #define CH1_WAKE_TCS_STATUS BIT(2) #define CH1_SLEEP_TCS_STATUS BIT(3) #define CH_CLEAR_STATUS BIT(31) #define ACCL_TYPE(addr) ((addr >> 16) & 0xF) #define VREG_ADDR(addr) (addr & ~0xF) #define MAX_RSC_COUNT 5 enum { HW_ACCL_CLK = 0x3, HW_ACCL_VREG, HW_ACCL_BUS, }; static const char * const accl_str[] = { "", "", "", "CLK", "VREG", "BUS", }; static LIST_HEAD(rpmh_rsc_dev_list); static struct rsc_drv *__rsc_drv[MAX_RSC_COUNT]; static int __rsc_count; bool rpmh_standalone; /* * Here's a high level overview of how all the registers in RPMH work * together: * * - The main rpmh-rsc address is the base of a register space that can * be used to find overall configuration of the hardware * (DRV_PRNT_CHLD_CONFIG). Also found within the rpmh-rsc register * space are all the TCS blocks. The offset of the TCS blocks is * specified in the device tree by "qcom,tcs-offset" and used to * compute tcs_base. * - TCS blocks come one after another. Type, count, and order are * specified by the device tree as "qcom,tcs-config". * - Each TCS block has some registers, then space for up to 16 commands. * Note that though address space is reserved for 16 commands, fewer * might be present. See ncpt (num cmds per TCS). * * Here's a picture: * * +---------------------------------------------------+ * |RSC | * | ctrl | * | | * | Drvs: | * | +-----------------------------------------------+ | * | |DRV0 | | * | | ctrl/config | | * | | IRQ | | * | | | | * | | TCSes: | | * | | +------------------------------------------+ | | * | | |TCS0 | | | | | | | | | | | | | | | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | * | | | | | | | | | | | | | | | | | | * | | +------------------------------------------+ | | * | | +------------------------------------------+ | | * | | |TCS1 | | | | | | | | | | | | | | | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | * | | | | | | | | | | | | | | | | | | * | | +------------------------------------------+ | | * | | +------------------------------------------+ | | * | | |TCS2 | | | | | | | | | | | | | | | * | | | ctrl | 0| 1| 2| 3| 4| 5| .| .| .| .|14|15| | | * | | | | | | | | | | | | | | | | | | * | | +------------------------------------------+ | | * | | ...... | | * | +-----------------------------------------------+ | * | +-----------------------------------------------+ | * | |DRV1 | | * | | (same as DRV0) | | * | +-----------------------------------------------+ | * | ...... | * +---------------------------------------------------+ */ #define USECS_TO_CYCLES(time_usecs) \ xloops_to_cycles((time_usecs) * 0x10C7UL) static inline unsigned long xloops_to_cycles(u64 xloops) { return (xloops * loops_per_jiffy * HZ) >> 32; } static u32 rpmh_rsc_reg_offset_ver_2_7[] = { [RSC_DRV_TCS_OFFSET] = 672, [RSC_DRV_CMD_OFFSET] = 20, [DRV_SOLVER_CONFIG] = 0x04, [DRV_PRNT_CHLD_CONFIG] = 0x0C, [RSC_DRV_IRQ_ENABLE] = 0x00, [RSC_DRV_IRQ_STATUS] = 0x04, [RSC_DRV_IRQ_CLEAR] = 0x08, [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x10, [RSC_DRV_CONTROL] = 0x14, [RSC_DRV_STATUS] = 0x18, [RSC_DRV_CMD_ENABLE] = 0x1C, [RSC_DRV_CMD_MSGID] = 0x30, [RSC_DRV_CMD_ADDR] = 0x34, [RSC_DRV_CMD_DATA] = 0x38, [RSC_DRV_CMD_STATUS] = 0x3C, [RSC_DRV_CMD_RESP_DATA] = 0x40, [RSC_DRV_CHN_SEQ_BUSY] = 0x0, [RSC_DRV_CHN_SEQ_PC] = 0x0, [RSC_DRV_CHN_TCS_TRIGGER] = 0x0, [RSC_DRV_CHN_TCS_COMPLETE] = 0x0, [RSC_DRV_CHN_UPDATE] = 0x0, [RSC_DRV_CHN_BUSY] = 0x0, [RSC_DRV_CHN_EN] = 0x0, }; static u32 rpmh_rsc_reg_offset_ver_3_0[] = { [RSC_DRV_TCS_OFFSET] = 672, [RSC_DRV_CMD_OFFSET] = 24, [DRV_SOLVER_CONFIG] = 0x04, [DRV_PRNT_CHLD_CONFIG] = 0x0C, [RSC_DRV_IRQ_ENABLE] = 0x00, [RSC_DRV_IRQ_STATUS] = 0x04, [RSC_DRV_IRQ_CLEAR] = 0x08, [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20, [RSC_DRV_CONTROL] = 0x24, [RSC_DRV_STATUS] = 0x28, [RSC_DRV_CMD_ENABLE] = 0x2C, [RSC_DRV_CMD_MSGID] = 0x34, [RSC_DRV_CMD_ADDR] = 0x38, [RSC_DRV_CMD_DATA] = 0x3C, [RSC_DRV_CMD_STATUS] = 0x40, [RSC_DRV_CMD_RESP_DATA] = 0x44, [RSC_DRV_CHN_SEQ_BUSY] = 0x464, [RSC_DRV_CHN_SEQ_PC] = 0x468, [RSC_DRV_CHN_TCS_TRIGGER] = 0x490, [RSC_DRV_CHN_TCS_COMPLETE] = 0x494, [RSC_DRV_CHN_UPDATE] = 0x498, [RSC_DRV_CHN_BUSY] = 0x49C, [RSC_DRV_CHN_EN] = 0x4A0, }; static u32 rpmh_rsc_reg_offset_ver_3_0_hw_channel[] = { [RSC_DRV_TCS_OFFSET] = 336, [RSC_DRV_CMD_OFFSET] = 24, [DRV_SOLVER_CONFIG] = 0x04, [DRV_PRNT_CHLD_CONFIG] = 0x0C, [RSC_DRV_IRQ_ENABLE] = 0x00, [RSC_DRV_IRQ_STATUS] = 0x04, [RSC_DRV_IRQ_CLEAR] = 0x08, [RSC_DRV_CMD_WAIT_FOR_CMPL] = 0x20, [RSC_DRV_CONTROL] = 0x24, [RSC_DRV_STATUS] = 0x28, [RSC_DRV_CMD_ENABLE] = 0x2C, [RSC_DRV_CMD_MSGID] = 0x34, [RSC_DRV_CMD_ADDR] = 0x38, [RSC_DRV_CMD_DATA] = 0x3C, [RSC_DRV_CMD_STATUS] = 0x40, [RSC_DRV_CMD_RESP_DATA] = 0x44, [RSC_DRV_CHN_SEQ_BUSY] = 0x464, [RSC_DRV_CHN_SEQ_PC] = 0x468, [RSC_DRV_CHN_TCS_TRIGGER] = 0x490, [RSC_DRV_CHN_TCS_COMPLETE] = 0x494, [RSC_DRV_CHN_UPDATE] = 0x498, [RSC_DRV_CHN_BUSY] = 0x49C, [RSC_DRV_CHN_EN] = 0x4A0, }; static inline void __iomem * tcs_reg_addr(const struct rsc_drv *drv, int reg, int tcs_id) { if (!drv->tcs_distance) return drv->tcs_base + drv->regs[RSC_DRV_TCS_OFFSET] * tcs_id + reg; return drv->tcs_base + drv->tcs_distance * tcs_id + reg; } static inline void __iomem * tcs_cmd_addr(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) { return tcs_reg_addr(drv, reg, tcs_id) + drv->regs[RSC_DRV_CMD_OFFSET] * cmd_id; } static u32 read_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id) { return readl_relaxed(tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); } static u32 read_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id) { return readl_relaxed(tcs_reg_addr(drv, reg, tcs_id)); } static void write_tcs_cmd(const struct rsc_drv *drv, int reg, int tcs_id, int cmd_id, u32 data) { writel_relaxed(data, tcs_cmd_addr(drv, reg, tcs_id, cmd_id)); } static void write_tcs_reg(const struct rsc_drv *drv, int reg, int tcs_id, u32 data) { writel_relaxed(data, tcs_reg_addr(drv, reg, tcs_id)); } static void write_tcs_reg_sync(const struct rsc_drv *drv, int reg, int tcs_id, u32 data) { int i; writel(data, tcs_reg_addr(drv, reg, tcs_id)); /* * Wait until we read back the same value. Use a counter rather than * ktime for timeout since this may be called after timekeeping stops. */ for (i = 0; i < USEC_PER_SEC; i++) { if (readl(tcs_reg_addr(drv, reg, tcs_id)) == data) return; udelay(1); } pr_err("%s: error writing %#x to %d:%#x\n", drv->name, data, tcs_id, reg); } /** * tcs_invalidate() - Invalidate all TCSes of the given type (sleep or wake). * @drv: The RSC controller. * @type: SLEEP_TCS or WAKE_TCS * @ch: Channel number * * This will clear the "slots" variable of the given tcs_group and also * tell the hardware to forget about all entries. * * The caller must ensure that no other RPMH actions are happening when this * function is called, since otherwise the device may immediately become * used again even before this function exits. */ static void tcs_invalidate(struct rsc_drv *drv, int type, int ch) { int m; struct tcs_group *tcs = &drv->ch[ch].tcs[type]; /* Caller ensures nobody else is running so no lock */ if (bitmap_empty(tcs->slots, tcs->ncpt * tcs->num_tcs)) return; for (m = tcs->offset; m < tcs->offset + tcs->num_tcs; m++) { write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], m, 0); write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], m, 0); } bitmap_zero(tcs->slots, tcs->ncpt * tcs->num_tcs); } /** * rpmh_rsc_get_channel() - Get the Unused channel to send data on. * @drv: The RSC controller. * * Return: 0 on success, else -error. */ int rpmh_rsc_get_channel(struct rsc_drv *drv) { int chn_update, chn_busy; if (drv->num_channels == 1) return CH0; /* Select Unused channel */ do { chn_update = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_UPDATE]); chn_busy = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_BUSY]); } while (chn_busy != chn_update); if (chn_busy & CH0_CHN_BUSY) return CH1; else if (chn_busy & CH1_CHN_BUSY) return CH0; else return -EBUSY; } /** * rpmh_rsc_invalidate() - Invalidate sleep and wake TCSes. * @drv: The RSC controller. * @ch: Channel number * * The caller must ensure that no other RPMH actions are happening when this * function is called, since otherwise the device may immediately become * used again even before this function exits. */ void rpmh_rsc_invalidate(struct rsc_drv *drv, int ch) { tcs_invalidate(drv, SLEEP_TCS, ch); tcs_invalidate(drv, WAKE_TCS, ch); } /** * get_tcs_for_msg() - Get the tcs_group used to send the given message. * @drv: The RSC controller. * @msg: The message we want to send. * * This is normally pretty straightforward except if we are trying to send * an ACTIVE_ONLY message but don't have any active_only TCSes. * * Return: A pointer to a tcs_group or an ERR_PTR. */ static struct tcs_group *get_tcs_for_msg(struct rsc_drv *drv, enum rpmh_state state, int ch) { int type; struct tcs_group *tcs; switch (state) { case RPMH_ACTIVE_ONLY_STATE: type = ACTIVE_TCS; break; case RPMH_WAKE_ONLY_STATE: type = WAKE_TCS; break; case RPMH_SLEEP_STATE: type = SLEEP_TCS; break; default: return ERR_PTR(-EINVAL); } /* * If we are making an active request on a RSC that does not have a * dedicated TCS for active state use, then re-purpose a wake TCS to * send active votes. This is safe because we ensure any active-only * transfers have finished before we use it (maybe by running from * the last CPU in PM code). */ tcs = &drv->ch[ch].tcs[type]; if (state == RPMH_ACTIVE_ONLY_STATE && !tcs->num_tcs) tcs = &drv->ch[ch].tcs[WAKE_TCS]; return tcs; } /** * get_req_from_tcs() - Get a stashed request that was xfering on the given TCS. * @drv: The RSC controller. * @tcs_id: The global ID of this TCS. * * For ACTIVE_ONLY transfers we want to call back into the client when the * transfer finishes. To do this we need the "request" that the client * originally provided us. This function grabs the request that we stashed * when we started the transfer. * * This only makes sense for ACTIVE_ONLY transfers since those are the only * ones we track sending (the only ones we enable interrupts for and the only * ones we call back to the client for). * * Return: The stashed request. */ static const struct tcs_request *get_req_from_tcs(struct rsc_drv *drv, int tcs_id, int *ch) { struct tcs_group *tcs; int i; for (i = 0; i < MAX_CHANNEL; i++) { if (!drv->ch[i].initialized) continue; tcs = get_tcs_for_msg(drv, RPMH_ACTIVE_ONLY_STATE, i); if (tcs->mask & BIT(tcs_id)) { *ch = i; return tcs->req[tcs_id - tcs->offset]; } } return NULL; } /** * __tcs_set_trigger() - Start xfer on a TCS or unset trigger on a borrowed TCS * @drv: The controller. * @tcs_id: The global ID of this TCS. * @trigger: If true then untrigger/retrigger. If false then just untrigger. * * In the normal case we only ever call with "trigger=true" to start a * transfer. That will un-trigger/disable the TCS from the last transfer * then trigger/enable for this transfer. * * If we borrowed a wake TCS for an active-only transfer we'll also call * this function with "trigger=false" to just do the un-trigger/disable * before using the TCS for wake purposes again. * * Note that the AP is only in charge of triggering active-only transfers. * The AP never triggers sleep/wake values using this function. */ static void __tcs_set_trigger(struct rsc_drv *drv, int tcs_id, bool trigger) { u32 enable; u32 reg = drv->regs[RSC_DRV_CONTROL]; /* * HW req: Clear the DRV_CONTROL and enable TCS again * While clearing ensure that the AMC mode trigger is cleared * and then the mode enable is cleared. */ enable = read_tcs_reg(drv, reg, tcs_id); enable &= ~TCS_AMC_MODE_TRIGGER; write_tcs_reg_sync(drv, reg, tcs_id, enable); enable &= ~TCS_AMC_MODE_ENABLE; write_tcs_reg_sync(drv, reg, tcs_id, enable); if (trigger) { /* Enable the AMC mode on the TCS and then trigger the TCS */ enable = TCS_AMC_MODE_ENABLE; write_tcs_reg_sync(drv, reg, tcs_id, enable); enable |= TCS_AMC_MODE_TRIGGER; write_tcs_reg(drv, reg, tcs_id, enable); } } /** * enable_tcs_irq() - Enable or disable interrupts on the given TCS. * @drv: The controller. * @tcs_id: The global ID of this TCS. * @enable: If true then enable; if false then disable * * We only ever call this when we borrow a wake TCS for an active-only * transfer. For active-only TCSes interrupts are always left enabled. */ static void enable_tcs_irq(struct rsc_drv *drv, int tcs_id, bool enable) { u32 data; u32 reg = drv->regs[RSC_DRV_IRQ_ENABLE]; data = readl_relaxed(drv->tcs_base + reg); if (enable) data |= BIT(tcs_id); else data &= ~BIT(tcs_id); writel_relaxed(data, drv->tcs_base + reg); } /** * tcs_tx_done() - TX Done interrupt handler. * @irq: The IRQ number (ignored). * @p: Pointer to "struct rsc_drv". * * Called for ACTIVE_ONLY transfers (those are the only ones we enable the * IRQ for) when a transfer is done. * * Return: IRQ_HANDLED */ static irqreturn_t tcs_tx_done(int irq, void *p) { struct rsc_drv *drv = p; int i, ch; unsigned long irq_status; const struct tcs_request *req; irq_status = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); for_each_set_bit(i, &irq_status, BITS_PER_TYPE(u32)) { req = get_req_from_tcs(drv, i, &ch); if (WARN_ON(!req)) goto skip; trace_rpmh_tx_done(drv, i, req); #if IS_ENABLED(CONFIG_IPC_LOGGING) ipc_log_string(drv->ipc_log_ctx, "IRQ response: m=%d", i); #endif /* * If wake tcs was re-purposed for sending active * votes, clear AMC trigger & enable modes and * disable interrupt for this TCS */ if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs) __tcs_set_trigger(drv, i, false); skip: /* Reclaim the TCS */ write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i, 0); write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], i, 0); writel_relaxed(BIT(i), drv->tcs_base + drv->regs[RSC_DRV_IRQ_CLEAR]); spin_lock(&drv->lock); clear_bit(i, drv->tcs_in_use); /* * Disable interrupt for WAKE TCS to avoid being * spammed with interrupts coming when the solver * sends its wake votes. */ if (!drv->ch[ch].tcs[ACTIVE_TCS].num_tcs) enable_tcs_irq(drv, i, false); spin_unlock(&drv->lock); wake_up(&drv->tcs_wait); if (req) rpmh_tx_done(req); } return IRQ_HANDLED; } /** * __tcs_buffer_write() - Write to TCS hardware from a request; don't trigger. * @drv: The controller. * @tcs_id: The global ID of this TCS. * @cmd_id: The index within the TCS to start writing. * @msg: The message we want to send, which will contain several addr/data * pairs to program (but few enough that they all fit in one TCS). * * This is used for all types of transfers (active, sleep, and wake). */ static void __tcs_buffer_write(struct rsc_drv *drv, int tcs_id, int cmd_id, const struct tcs_request *msg) { u32 msgid; u32 cmd_msgid = CMD_MSGID_LEN | CMD_MSGID_WRITE; u32 cmd_enable = 0; u32 cmd_complete; struct tcs_cmd *cmd; int i, j; /* Convert all commands to RR when the request has wait_for_compl set */ cmd_msgid |= msg->wait_for_compl ? CMD_MSGID_RESP_REQ : 0; cmd_complete = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], tcs_id); for (i = 0, j = cmd_id; i < msg->num_cmds; i++, j++) { cmd = &msg->cmds[i]; cmd_enable |= BIT(j); cmd_complete |= cmd->wait << j; msgid = cmd_msgid; /* * Additionally, if the cmd->wait is set, make the command * response reqd even if the overall request was fire-n-forget. */ msgid |= cmd->wait ? CMD_MSGID_RESP_REQ : 0; write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, j, msgid); write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, j, cmd->addr); write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, j, cmd->data); trace_rpmh_send_msg(drv, tcs_id, msg->state, j, msgid, cmd); #if IS_ENABLED(CONFIG_IPC_LOGGING) ipc_log_string(drv->ipc_log_ctx, "TCS write: m=%d n=%d msgid=%#x addr=%#x data=%#x wait=%d", tcs_id, j, msgid, cmd->addr, cmd->data, cmd->wait); #endif } write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], tcs_id, cmd_complete); cmd_enable |= read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); write_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, cmd_enable); } /** * check_for_req_inflight() - Look to see if conflicting cmds are in flight. * @drv: The controller. * @tcs: A pointer to the tcs_group used for ACTIVE_ONLY transfers. * @msg: The message we want to send, which will contain several addr/data * pairs to program (but few enough that they all fit in one TCS). * * This will walk through the TCSes in the group and check if any of them * appear to be sending to addresses referenced in the message. If it finds * one it'll return -EBUSY. * * Only for use for active-only transfers. * * Must be called with the drv->lock held since that protects tcs_in_use. * * Return: 0 if nothing in flight or -EBUSY if we should try again later. * The caller must re-enable interrupts between tries since that's * the only way tcs_in_use will ever be updated and the only way * RSC_DRV_CMD_ENABLE will ever be cleared. */ static int check_for_req_inflight(struct rsc_drv *drv, struct tcs_group *tcs, const struct tcs_request *msg) { unsigned long curr_enabled; u32 addr; int j, k; int i = tcs->offset; unsigned long accl; for_each_set_bit_from(i, drv->tcs_in_use, tcs->offset + tcs->num_tcs) { curr_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], i); for_each_set_bit(j, &curr_enabled, tcs->ncpt) { addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], i, j); for (k = 0; k < msg->num_cmds; k++) { /* * Each RPMh VREG accelerator resource has 3 or 4 contiguous 4-byte * aligned addresses associated with it. Ignore the offset to check * for in-flight VREG requests. */ accl = ACCL_TYPE(msg->cmds[k].addr); if (accl == HW_ACCL_VREG && VREG_ADDR(addr) == VREG_ADDR(msg->cmds[k].addr)) return -EBUSY; else if (addr == msg->cmds[k].addr) return -EBUSY; } } } return 0; } /** * find_free_tcs() - Find free tcs in the given tcs_group; only for active. * @tcs: A pointer to the active-only tcs_group (or the wake tcs_group if * we borrowed it because there are zero active-only ones). * * Must be called with the drv->lock held since that protects tcs_in_use. * * Return: The first tcs that's free or -EBUSY if all in use. */ static int find_free_tcs(struct tcs_group *tcs) { const struct rsc_drv *drv = tcs->drv; unsigned long i; unsigned long max = tcs->offset + tcs->num_tcs; int timeout = 100; i = find_next_zero_bit(drv->tcs_in_use, max, tcs->offset); if (i >= max) return -EBUSY; while (timeout) { if (read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], i)) break; timeout--; udelay(1); } if (!timeout) return -EBUSY; return i; } /** * claim_tcs_for_req() - Claim a tcs in the given tcs_group; only for active. * @drv: The controller. * @tcs: The tcs_group used for ACTIVE_ONLY transfers. * @msg: The data to be sent. * * Claims a tcs in the given tcs_group while making sure that no existing cmd * is in flight that would conflict with the one in @msg. * * Context: Must be called with the drv->lock held since that protects * tcs_in_use. * * Return: The id of the claimed tcs or -EBUSY if a matching msg is in flight * or the tcs_group is full. */ static int claim_tcs_for_req(struct rsc_drv *drv, struct tcs_group *tcs, const struct tcs_request *msg) { int ret; /* * The h/w does not like if we send a request to the same address, * when one is already in-flight or being processed. */ ret = check_for_req_inflight(drv, tcs, msg); if (ret) return ret; return find_free_tcs(tcs); } /** * rpmh_rsc_send_data() - Write / trigger active-only message. * @drv: The controller. * @msg: The data to be sent. * @ch: Channel number * * NOTES: * - This is only used for "ACTIVE_ONLY" since the limitations of this * function don't make sense for sleep/wake cases. * - To do the transfer, we will grab a whole TCS for ourselves--we don't * try to share. If there are none available we'll wait indefinitely * for a free one. * - This function will not wait for the commands to be finished, only for * data to be programmed into the RPMh. See rpmh_tx_done() which will * be called when the transfer is fully complete. * - This function must be called with interrupts enabled. If the hardware * is busy doing someone else's transfer we need that transfer to fully * finish so that we can have the hardware, and to fully finish it needs * the interrupt handler to run. If the interrupts is set to run on the * active CPU this can never happen if interrupts are disabled. * * Return: 0 on success, -EINVAL on error. */ int rpmh_rsc_send_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { struct tcs_group *tcs; int tcs_id; unsigned long flags; tcs = get_tcs_for_msg(drv, msg->state, ch); if (IS_ERR(tcs)) return PTR_ERR(tcs); spin_lock_irqsave(&drv->lock, flags); /* Controller is busy in 'solver' mode */ if (drv->in_solver_mode) { spin_unlock_irqrestore(&drv->lock, flags); return -EBUSY; } /* Wait forever for a free tcs. It better be there eventually! */ wait_event_lock_irq(drv->tcs_wait, (tcs_id = claim_tcs_for_req(drv, tcs, msg)) >= 0, drv->lock); tcs->req[tcs_id - tcs->offset] = msg; set_bit(tcs_id, drv->tcs_in_use); /* * Clear previously programmed ACTIVE/WAKE commands in selected * repurposed TCS to avoid triggering them. tcs->slots will be * cleaned from rpmh_flush() by invoking rpmh_rsc_invalidate() */ write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id, 0); write_tcs_reg_sync(drv, drv->regs[RSC_DRV_CMD_WAIT_FOR_CMPL], tcs_id, 0); if (msg->wait_for_compl || (msg->state == RPMH_ACTIVE_ONLY_STATE && tcs->type != ACTIVE_TCS)) enable_tcs_irq(drv, tcs_id, true); else enable_tcs_irq(drv, tcs_id, false); /* * These two can be done after the lock is released because: * - We marked "tcs_in_use" under lock. * - Once "tcs_in_use" has been marked nobody else could be writing * to these registers until the interrupt goes off. * - The interrupt can't go off until we trigger w/ the last line * of __tcs_set_trigger() below. */ __tcs_buffer_write(drv, tcs_id, 0, msg); __tcs_set_trigger(drv, tcs_id, true); #if IS_ENABLED(CONFIG_IPC_LOGGING) ipc_log_string(drv->ipc_log_ctx, "TCS trigger: m=%d wait_for_compl=%u", tcs_id, msg->wait_for_compl); #endif if (!msg->wait_for_compl) clear_bit(tcs_id, drv->tcs_in_use); spin_unlock_irqrestore(&drv->lock, flags); if (!msg->wait_for_compl) wake_up(&drv->tcs_wait); return 0; } /** * find_slots() - Find a place to write the given message. * @tcs: The tcs group to search. * @msg: The message we want to find room for. * @tcs_id: If we return 0 from the function, we return the global ID of the * TCS to write to here. * @cmd_id: If we return 0 from the function, we return the index of * the command array of the returned TCS where the client should * start writing the message. * * Only for use on sleep/wake TCSes since those are the only ones we maintain * tcs->slots for. * * Return: -ENOMEM if there was no room, else 0. */ static int find_slots(struct tcs_group *tcs, const struct tcs_request *msg, int *tcs_id, int *cmd_id) { int slot, offset; int i = 0; /* Do over, until we can fit the full payload in a single TCS */ do { slot = bitmap_find_next_zero_area(tcs->slots, tcs->ncpt * tcs->num_tcs, i, msg->num_cmds, 0); if (slot >= tcs->num_tcs * tcs->ncpt) return -ENOMEM; i += tcs->ncpt; } while (slot + msg->num_cmds - 1 >= i); bitmap_set(tcs->slots, slot, msg->num_cmds); offset = slot / tcs->ncpt; *tcs_id = offset + tcs->offset; *cmd_id = slot % tcs->ncpt; return 0; } /** * rpmh_rsc_write_ctrl_data() - Write request to controller but don't trigger. * @drv: The controller. * @msg: The data to be written to the controller. * @ch: Channel number * * This should only be called for sleep/wake state, never active-only * state. * * The caller must ensure that no other RPMH actions are happening and the * controller is idle when this function is called since it runs lockless. * * Return: 0 if no error; else -error. */ int rpmh_rsc_write_ctrl_data(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { struct tcs_group *tcs; int tcs_id = 0, cmd_id = 0; int ret; if (!msg->num_cmds) { #if IS_ENABLED(CONFIG_IPC_LOGGING) ipc_log_string(drv->ipc_log_ctx, "Empty num_cmds, returning"); #endif return 0; } tcs = get_tcs_for_msg(drv, msg->state, ch); if (IS_ERR(tcs)) return PTR_ERR(tcs); /* find the TCS id and the command in the TCS to write to */ ret = find_slots(tcs, msg, &tcs_id, &cmd_id); if (!ret) __tcs_buffer_write(drv, tcs_id, cmd_id, msg); return ret; } static struct tcs_group *get_tcs_from_index(struct rsc_drv *drv, int tcs_id) { unsigned int i, j; for (i = 0; i < TCS_TYPE_NR; i++) { for (j = 0; j < MAX_CHANNEL; j++) { if (!drv->ch[j].initialized) continue; if (drv->ch[j].tcs[i].mask & BIT(tcs_id)) return &drv->ch[j].tcs[i]; } } return NULL; } static void print_tcs_info(struct rsc_drv *drv, int tcs_id, unsigned long *accl, bool *aoss_irq_sts) { int ch = 0; struct tcs_group *tcs_grp = get_tcs_from_index(drv, tcs_id); const struct tcs_request *req = get_req_from_tcs(drv, tcs_id, &ch); unsigned long cmds_enabled; u32 addr, data, msgid, sts, irq_sts; bool in_use = test_bit(tcs_id, drv->tcs_in_use); int i; sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); cmds_enabled = read_tcs_reg(drv, drv->regs[RSC_DRV_CMD_ENABLE], tcs_id); if (!cmds_enabled || !tcs_grp) return; if (!req) goto print_tcs_data; data = read_tcs_reg(drv, drv->regs[RSC_DRV_CONTROL], tcs_id); irq_sts = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_STATUS]); pr_warn("Request: tcs-in-use:%s active_tcs=%s(%d) state=%d wait_for_compl=%u]\n", (in_use ? "YES" : "NO"), ((tcs_grp->type == ACTIVE_TCS) ? "YES" : "NO"), tcs_grp->type, req->state, req->wait_for_compl); pr_warn("TCS=%d [ctrlr-sts:%s amc-mode:0x%x irq-sts:%s]\n", tcs_id, sts ? "IDLE" : "BUSY", data, (irq_sts & BIT(tcs_id)) ? "COMPLETED" : "PENDING"); *aoss_irq_sts = (irq_sts & BIT(tcs_id)) ? true : false; print_tcs_data: for_each_set_bit(i, &cmds_enabled, tcs_grp->ncpt) { addr = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_ADDR], tcs_id, i); data = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i); msgid = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_MSGID], tcs_id, i); sts = read_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_STATUS], tcs_id, i); pr_warn("\tCMD=%d [addr=0x%x data=0x%x hdr=0x%x sts=0x%x enabled=1]\n", i, addr, data, msgid, sts); if (!(sts & CMD_STATUS_ISSUED)) continue; if (!(sts & CMD_STATUS_COMPL)) *accl |= BIT(ACCL_TYPE(addr)); } } void rpmh_rsc_debug_channel_busy(struct rsc_drv *drv) { u32 event_sts, ctrl_sts; u32 chn_update, chn_busy, chn_en; u32 seq_busy, seq_pc; pr_err("RSC:%s\n", drv->name); event_sts = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_TCS_COMPLETE]); ctrl_sts = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_TCS_TRIGGER]); chn_update = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_UPDATE]); chn_busy = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_BUSY]); chn_en = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_EN]); seq_busy = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_SEQ_BUSY]); seq_pc = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_SEQ_PC]); pr_err("event sts: 0x%x ctrl_sts: 0x%x\n", event_sts, ctrl_sts); pr_err("chn_update: 0x%x chn_busy: 0x%x chn_en: 0x%x\n", chn_update, chn_busy, chn_en); pr_err("seq_busy: 0x%x seq_pc: 0x%x\n", seq_busy, seq_pc); crm_dump_regs("cam_crm"); crm_dump_drv_regs("cam_crm", drv->id); } void rpmh_rsc_debug(struct rsc_drv *drv, struct completion *compl) { struct irq_data *rsc_irq_data = irq_get_irq_data(drv->irq); bool gic_irq_sts, aoss_irq_sts = false; int i; int busy = 0; unsigned long accl = 0; char str[20] = ""; pr_warn("RSC:%s\n", drv->name); for (i = 0; i < drv->num_tcs; i++) { if (!test_bit(i, drv->tcs_in_use)) continue; busy++; print_tcs_info(drv, i, &accl, &aoss_irq_sts); } if (!rsc_irq_data) { pr_err("No IRQ data for RSC:%s\n", drv->name); return; } irq_get_irqchip_state(drv->irq, IRQCHIP_STATE_PENDING, &gic_irq_sts); pr_warn("HW IRQ %lu is %s at GIC\n", rsc_irq_data->hwirq, gic_irq_sts ? "PENDING" : "NOT PENDING"); pr_warn("Completion is %s to finish\n", completion_done(compl) ? "PENDING" : "NOT PENDING"); for_each_set_bit(i, &accl, ARRAY_SIZE(accl_str)) { strlcat(str, accl_str[i], sizeof(str)); strlcat(str, " ", sizeof(str)); } if ((busy && !gic_irq_sts) || !aoss_irq_sts) pr_warn("ERROR:Accelerator(s) { %s } at AOSS did not respond\n", str); else if (gic_irq_sts) pr_warn("ERROR:Possible lockup in Linux\n"); /* Show fast path status, if the TCS is busy */ for (i = 0; i < MAX_CHANNEL; i++) { if (!drv->ch[i].initialized) continue; /* Show fast path status, if the TCS is busy */ if (drv->ch[i].tcs[FAST_PATH_TCS].num_tcs) { int tcs_id = drv->ch[i].tcs[FAST_PATH_TCS].offset; bool sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); if (!sts) { pr_err("Fast-path TCS information:\n"); print_tcs_info(drv, tcs_id, &accl, &aoss_irq_sts); } } } /* * The TCS(s) are busy waiting, we have no way to recover from this. * If this debug function is called, we assume it's because timeout * has happened. * Crash and report. */ BUG_ON(busy); } /** * rpmh_rsc_ctrlr_is_busy() - Check if any of the AMCs are busy. * @drv: The controller * * Checks if any of the AMCs are busy in handling ACTIVE sets. * This is called from the last cpu powering down before flushing * SLEEP and WAKE sets. If AMCs are busy, controller can not enter * power collapse, so deny from the last cpu's pm notification. * * Context: Must be called with the drv->lock held. * * Return: * * False - AMCs are idle * * True - AMCs are busy */ static bool rpmh_rsc_ctrlr_is_busy(struct rsc_drv *drv) { int i; struct tcs_group *tcs; unsigned long set; unsigned long max; for (i = 0; i < MAX_CHANNEL; i++) { if (!drv->ch[i].initialized) continue; tcs = &drv->ch[i].tcs[ACTIVE_TCS]; /* * If we made an active request on a RSC that does not have a * dedicated TCS for active state use, then re-purposed wake TCSes * should be checked for not busy, because we used wake TCSes for * active requests in this case. */ if (!tcs->num_tcs) tcs = &drv->ch[i].tcs[WAKE_TCS]; max = tcs->offset + tcs->num_tcs; set = find_next_bit(drv->tcs_in_use, max, tcs->offset); if (set < max) return true; /* Check if there is pending fastpath transaction */ tcs = &drv->ch[i].tcs[FAST_PATH_TCS]; if (tcs->num_tcs && !read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs->offset)) return true; } return false; } /** * rpmh_rsc_write_next_wakeup() - Write next wakeup in CONTROL_TCS. * @drv: The controller * * Writes maximum wakeup cycles when called from suspend. * Writes earliest hrtimer wakeup when called from idle. */ void rpmh_rsc_write_next_wakeup(struct rsc_drv *drv) { ktime_t now, wakeup; u64 wakeup_us, wakeup_cycles = ~0; u32 lo, hi; if (!drv->tcs[CONTROL_TCS].num_tcs || !drv->genpd_nb.notifier_call) return; /* Set highest time when system (timekeeping) is suspended */ if (system_state == SYSTEM_SUSPEND) goto exit; /* Find the earliest hrtimer wakeup from online cpus */ wakeup = dev_pm_genpd_get_next_hrtimer(drv->dev); /* Find the relative wakeup in kernel time scale */ now = ktime_get(); wakeup = ktime_sub(wakeup, now); wakeup_us = ktime_to_us(wakeup); /* Convert the wakeup to arch timer scale */ wakeup_cycles = USECS_TO_CYCLES(wakeup_us); wakeup_cycles += arch_timer_read_counter(); exit: lo = wakeup_cycles & RSC_DRV_CTL_TCS_DATA_LO_MASK; hi = wakeup_cycles >> RSC_DRV_CTL_TCS_DATA_SIZE; hi &= RSC_DRV_CTL_TCS_DATA_HI_MASK; hi |= RSC_DRV_CTL_TCS_DATA_HI_VALID; writel_relaxed(lo, drv->base + RSC_DRV_CTL_TCS_DATA_LO); writel_relaxed(hi, drv->base + RSC_DRV_CTL_TCS_DATA_HI); } /** * rpmh_rsc_cpu_pm_callback() - Check if any of the AMCs are busy. * @nfb: Pointer to the notifier block in struct rsc_drv. * @action: CPU_PM_ENTER, CPU_PM_ENTER_FAILED, or CPU_PM_EXIT. * @v: Unused * * This function is given to cpu_pm_register_notifier so we can be informed * about when CPUs go down. When all CPUs go down we know no more active * transfers will be started so we write sleep/wake sets. This function gets * called from cpuidle code paths and also at system suspend time. * * If its last CPU going down and AMCs are not busy then writes cached sleep * and wake messages to TCSes. The firmware then takes care of triggering * them when entering deepest low power modes. * * Return: See cpu_pm_register_notifier() */ static int rpmh_rsc_cpu_pm_callback(struct notifier_block *nfb, unsigned long action, void *v) { struct rsc_drv *drv = container_of(nfb, struct rsc_drv, rsc_pm); int ret = NOTIFY_OK; int cpus_in_pm, ch; switch (action) { case CPU_PM_ENTER: cpus_in_pm = atomic_inc_return(&drv->cpus_in_pm); /* * NOTE: comments for num_online_cpus() point out that it's * only a snapshot so we need to be careful. It should be OK * for us to use, though. It's important for us not to miss * if we're the last CPU going down so it would only be a * problem if a CPU went offline right after we did the check * AND that CPU was not idle AND that CPU was the last non-idle * CPU. That can't happen. CPUs would have to come out of idle * before the CPU could go offline. */ if (cpus_in_pm < num_online_cpus()) return NOTIFY_OK; break; case CPU_PM_ENTER_FAILED: case CPU_PM_EXIT: atomic_dec(&drv->cpus_in_pm); return NOTIFY_OK; default: return NOTIFY_DONE; } /* * It's likely we're on the last CPU. Grab the drv->lock and write * out the sleep/wake commands to RPMH hardware. Grabbing the lock * means that if we race with another CPU coming up we are still * guaranteed to be safe. If another CPU came up just after we checked * and has grabbed the lock or started an active transfer then we'll * notice we're busy and abort. If another CPU comes up after we start * flushing it will be blocked from starting an active transfer until * we're done flushing. If another CPU starts an active transfer after * we release the lock we're still OK because we're no longer the last * CPU. */ if (spin_trylock(&drv->lock)) { ch = rpmh_rsc_get_channel(drv); if (ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || rpmh_flush(&drv->client, ch)) ret = NOTIFY_BAD; spin_unlock(&drv->lock); } else { /* Another CPU must be up */ return NOTIFY_OK; } if (ret == NOTIFY_BAD) { /* Double-check if we're here because someone else is up */ if (cpus_in_pm < num_online_cpus()) ret = NOTIFY_OK; else /* We won't be called w/ CPU_PM_ENTER_FAILED */ atomic_dec(&drv->cpus_in_pm); } return ret; } /** * rpmh_rsc_mode_solver_set() - Enable/disable solver mode. * @drv: The controller. * @enable: Boolean state to be set - true/false * * Return: * * 0 - success * * -EBUSY - AMCs are busy */ int rpmh_rsc_mode_solver_set(struct rsc_drv *drv, bool enable) { int ret = -EBUSY; if (spin_trylock(&drv->lock)) { if (!enable || !rpmh_rsc_ctrlr_is_busy(drv)) { drv->in_solver_mode = enable; trace_rpmh_solver_set(drv, enable); #if IS_ENABLED(CONFIG_IPC_LOGGING) ipc_log_string(drv->ipc_log_ctx, "solver mode set: %d", enable); #endif ret = 0; } spin_unlock(&drv->lock); } return ret; } int rpmh_rsc_is_tcs_completed(struct rsc_drv *drv, int ch) { u32 sts; int retry = 10, ret = 0; do { sts = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_TCS_COMPLETE]); if (ch == 0) sts &= CH0_WAKE_TCS_STATUS; else sts &= CH1_WAKE_TCS_STATUS; retry--; /* * Wait till all the WAKE votes of the new channel are * applied during channel switch. * Maximum delay of 100 usec. */ if (!sts) udelay(10); } while (!sts && retry); if (!retry) { ret = -EBUSY; goto exit; } writel_relaxed(CH_CLEAR_STATUS, drv->base + drv->regs[RSC_DRV_CHN_TCS_COMPLETE]); exit: trace_rpmh_switch_channel(drv, ch, ret); #if IS_ENABLED(CONFIG_IPC_LOGGING) ipc_log_string(drv->ipc_log_ctx, "channel switched to: %d ret: %d", ch, ret); #endif return 0; } /** * rpmh_rsc_switch_channel() - Switch to the channel * @drv: The controller. * @ch: The channel number to switch to. * * NOTE: Caller should ensure serialization before making this call. * Return: * * 0 - success * * -Error - Error code */ int rpmh_rsc_switch_channel(struct rsc_drv *drv, int ch) { writel_relaxed(BIT(ch), drv->base + drv->regs[RSC_DRV_CHN_UPDATE]); return rpmh_rsc_is_tcs_completed(drv, ch); } /** * rpmh_rsc_drv_enable() - Enable the DRV and trigger Wake vote * @drv: The controller. * * NOTE: Caller should ensure serialization before making this call. * Return: * * 0 - success * * -Error - Error code */ int rpmh_rsc_drv_enable(struct rsc_drv *drv, bool enable) { int ret = 0, ch; u32 chn_en; spin_lock(&drv->lock); chn_en = readl_relaxed(drv->base + drv->regs[RSC_DRV_CHN_EN]); if (chn_en == enable) { ret = -EINVAL; goto exit; } if (enable) { /* Start with channel 0 */ ch = 0; ret = rpmh_flush(&drv->client, ch); if (ret) goto exit; writel_relaxed(enable, drv->base + drv->regs[RSC_DRV_CHN_EN]); ret = rpmh_rsc_switch_channel(drv, ch); if (ret) goto exit; } else { /* Select unused channel */ ch = rpmh_rsc_get_channel(drv); if (ch < 0) goto exit; ret = rpmh_flush(&drv->client, ch); if (ret) goto exit; ret = rpmh_rsc_switch_channel(drv, ch); if (ret) goto exit; writel_relaxed(0, drv->base + drv->regs[RSC_DRV_CHN_UPDATE]); writel_relaxed(enable, drv->base + drv->regs[RSC_DRV_CHN_EN]); } exit: spin_unlock(&drv->lock); trace_rpmh_drv_enable(drv, enable, ret); #if IS_ENABLED(CONFIG_IPC_LOGGING) ipc_log_string(drv->ipc_log_ctx, "drv enable: %d ret: %d", enable, ret); #endif return ret; } /** * rpmh_rsc_init_fast_path() - Initialize the fast-path TCS contents * @drv: The controller. * @msg: The TCS request to populate. * @ch: Channel number * * Return: * * 0 - success * * -ENODEV - no fast-path TCS available */ int rpmh_rsc_init_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, int ch) { int tcs_id; if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs) return -ENODEV; tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset; /* We won't use the AMC IRQ to confirm if the TCS is free */ enable_tcs_irq(drv, tcs_id, false); __tcs_buffer_write(drv, tcs_id, 0, msg); return 0; } /** * rpmh_rsc_update_fast_path() - Update the fast-path TCS data and trigger * @drv: The controller. * @msg: The TCS request data to be updated. * @mask: The update mask for elements in @msg to be sent * @ch: Channel number * * NOTE: Caller should ensure serialization before making this call. * Return: * * 0 - success * * -ENODEV - no fast-path TCS available */ int rpmh_rsc_update_fast_path(struct rsc_drv *drv, const struct tcs_request *msg, u32 mask, int ch) { int i; u32 sts; int tcs_id; struct tcs_cmd *cmd; int retry = 5; if (!drv->ch[ch].tcs[FAST_PATH_TCS].num_tcs) return -ENODEV; tcs_id = drv->ch[ch].tcs[FAST_PATH_TCS].offset; /* Ensure the TCS is free before writing to the TCS */ do { sts = read_tcs_reg(drv, drv->regs[RSC_DRV_STATUS], tcs_id); if (!sts) { retry--; /* Report and bail, if it took too many attempts */ if (!retry) { pr_err("Fast-path TCS is too busy\n"); return -EBUSY; } udelay(1); } } while (!sts); /* * We only update the data, everything else remains the same. * The number of commands and the addresses do not change with * updates. */ for (i = 0; i < msg->num_cmds; i++) { if (!(mask & BIT(i))) continue; cmd = &msg->cmds[i]; write_tcs_cmd(drv, drv->regs[RSC_DRV_CMD_DATA], tcs_id, i, cmd->data); } /* Trigger the TCS to send the request */ __tcs_set_trigger(drv, tcs_id, true); return 0; } static int rpmh_rsc_poweroff_noirq(struct device *dev) { return 0; } static void rpmh_rsc_tcs_irq_enable(struct rsc_drv *drv) { u32 tcs_mask; int ch; for (ch = 0; ch < MAX_CHANNEL; ch++) { if (!drv->ch[ch].initialized) continue; tcs_mask = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); tcs_mask |= drv->ch[ch].tcs[ACTIVE_TCS].mask; writel_relaxed(tcs_mask, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); } } static int rpmh_rsc_restore_noirq(struct device *dev) { struct rsc_drv_top *rsc_top = dev_get_drvdata(dev); int i; for (i = 0; i < rsc_top->drv_count; i++) { if (rsc_top->drv[i].initialized) rpmh_rsc_tcs_irq_enable(&rsc_top->drv[i]); } return 0; } static struct rsc_drv_top *rpmh_rsc_get_top_device(const char *name) { struct rsc_drv_top *rsc_top; bool rsc_dev_present = false; list_for_each_entry(rsc_top, &rpmh_rsc_dev_list, list) { if (!strcmp(name, rsc_top->name)) { rsc_dev_present = true; break; } } if (!rsc_dev_present) return ERR_PTR(-ENODEV); return rsc_top; } const struct device *rpmh_rsc_get_device(const char *name, u32 drv_id) { struct rsc_drv_top *rsc_top = rpmh_rsc_get_top_device(name); int i; if (IS_ERR(rsc_top) || strcmp(name, "cam_rsc")) return ERR_PTR(-ENODEV); for (i = 0; i < rsc_top->drv_count; i++) { if (i == drv_id && rsc_top->drv[i].initialized) return rsc_top->drv[i].dev; } return ERR_PTR(-ENODEV); } static int rpmh_probe_channel_tcs_config(struct device_node *np, struct rsc_drv *drv, u32 max_tcs, u32 ncpt, int ch) { struct tcs_type_config { u32 type; u32 n; } tcs_cfg[TCS_TYPE_NR] = { { 0 } }; struct tcs_group *tcs; struct drv_channel *channel = &drv->ch[ch]; int i, ret, n, st = 0; u32 tcs_mask; n = of_property_count_u32_elems(np, "qcom,tcs-config"); if (n != 2 * TCS_TYPE_NR) return -EINVAL; for (i = 0; i < TCS_TYPE_NR; i++) { ret = of_property_read_u32_index(np, "qcom,tcs-config", i * 2, &tcs_cfg[i].type); if (ret) return ret; if (tcs_cfg[i].type >= TCS_TYPE_NR) return -EINVAL; ret = of_property_read_u32_index(np, "qcom,tcs-config", i * 2 + 1, &tcs_cfg[i].n); if (ret) return ret; if (tcs_cfg[i].n > MAX_TCS_PER_TYPE) return -EINVAL; } for (i = 0; i < TCS_TYPE_NR; i++) { tcs = &channel->tcs[tcs_cfg[i].type]; if (tcs->drv) return -EINVAL; tcs->drv = drv; tcs->type = tcs_cfg[i].type; tcs->num_tcs = tcs_cfg[i].n; tcs->ncpt = ncpt; if (!tcs->num_tcs || tcs->type == CONTROL_TCS) continue; if (st + tcs->num_tcs > max_tcs || st + tcs->num_tcs >= BITS_PER_BYTE * sizeof(tcs->mask)) return -EINVAL; tcs->mask = ((1 << tcs->num_tcs) - 1) << (st + drv->num_tcs); tcs->offset = st + drv->num_tcs; st += tcs->num_tcs; } /* Enable the active TCS to send requests immediately */ tcs_mask = readl_relaxed(drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); tcs_mask |= drv->ch[ch].tcs[ACTIVE_TCS].mask; writel_relaxed(tcs_mask, drv->tcs_base + drv->regs[RSC_DRV_IRQ_ENABLE]); channel->drv = drv; channel->initialized = true; drv->num_tcs += st; return 0; } /** * rpmh_rsc_pd_callback() - Check if any of the AMCs are busy. * @nfb: Pointer to the genpd notifier block in struct rsc_drv. * @action: GENPD_NOTIFY_PRE_OFF, GENPD_NOTIFY_OFF, GENPD_NOTIFY_PRE_ON or GENPD_NOTIFY_ON. * @v: Unused * * This function is given to dev_pm_genpd_add_notifier() so we can be informed * about when cluster-pd is going down. When cluster go down we know no more active * transfers will be started so we write sleep/wake sets. This function gets * called from cpuidle code paths and also at system suspend time. * * If AMCs are not busy then writes cached sleep and wake messages to TCSes. * The firmware then takes care of triggering them when entering deepest low power modes. * * Return: * * NOTIFY_OK - success * * NOTIFY_BAD - failure */ static int rpmh_rsc_pd_callback(struct notifier_block *nfb, unsigned long action, void *v) { struct rsc_drv *drv = container_of(nfb, struct rsc_drv, genpd_nb); int ch; ch = rpmh_rsc_get_channel(drv); /* We don't need to lock as genpd on/off are serialized */ if ((action == GENPD_NOTIFY_PRE_OFF) && (ch < 0 || rpmh_rsc_ctrlr_is_busy(drv) || _rpmh_flush(&drv->client, ch))) return NOTIFY_BAD; return NOTIFY_OK; } static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) { int ret; pm_runtime_enable(dev); drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback; ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb); if (ret) pm_runtime_disable(dev); return ret; } static int rpmh_probe_tcs_config(struct rsc_drv *drv) { struct device_node *cn, *np = drv->dev->of_node; int ch = 0, ret; u32 offset, config; u32 max_tcs, ncpt; ret = of_property_read_u32(np, "qcom,tcs-offset", &offset); if (ret) return ret; drv->tcs_base = drv->base + offset; ret = of_property_read_u32(np, "qcom,tcs-distance", &drv->tcs_distance); if (ret) drv->tcs_distance = 0; config = readl_relaxed(drv->base + drv->regs[DRV_PRNT_CHLD_CONFIG]); max_tcs = config; max_tcs &= DRV_NUM_TCS_MASK << (DRV_NUM_TCS_SHIFT * drv->id); max_tcs = max_tcs >> (DRV_NUM_TCS_SHIFT * drv->id); ncpt = config & (DRV_NCPT_MASK << DRV_NCPT_SHIFT); ncpt = ncpt >> DRV_NCPT_SHIFT; for_each_child_of_node(np, cn) { if (!of_node_name_eq(cn, "channel")) continue; ret = rpmh_probe_channel_tcs_config(cn, drv, max_tcs, ncpt, ch); if (ret) return ret; ch++; } drv->num_channels = ch; return 0; } static int rpmh_rsc_probe(struct platform_device *pdev) { struct device_node *np, *dn = pdev->dev.of_node; struct rsc_drv *drv; struct rsc_drv_top *rsc_top; int ret, irq; u32 rsc_id, major_ver, minor_ver, solver_config; int i, j, drv_count; const char *name; /* * Even though RPMh doesn't directly use cmd-db, all of its children * do. To avoid adding this check to our children we'll do it now. */ ret = cmd_db_ready(); if (ret) { if (ret != -EPROBE_DEFER) dev_err(&pdev->dev, "Command DB not available (%d)\n", ret); return ret; } rpmh_standalone = cmd_db_is_standalone(); if (rpmh_standalone) dev_info(&pdev->dev, "RPMH is running in standalone mode.\n"); rsc_top = devm_kzalloc(&pdev->dev, sizeof(*rsc_top), GFP_KERNEL); if (!rsc_top) return -ENOMEM; ret = of_property_read_u32(dn, "qcom,drv-count", &drv_count); if (ret) return ret; drv = devm_kcalloc(&pdev->dev, drv_count, sizeof(*drv), GFP_KERNEL); if (!drv) return -ENOMEM; name = of_get_property(dn, "label", NULL); if (!name) name = dev_name(&pdev->dev); rsc_top->drv_count = drv_count; rsc_top->drv = drv; rsc_top->dev = &pdev->dev; scnprintf(rsc_top->name, sizeof(rsc_top->name), "%s", name); for_each_child_of_node(dn, np) { struct device *drv_dev; if (!of_node_name_eq(np, "drv")) continue; ret = of_property_read_u32(np, "qcom,drv-id", &i); if (ret) return ret; scnprintf(drv[i].name, sizeof(drv[i].name), "%s-drv-%d", name, i); drv[i].base = devm_platform_ioremap_resource(pdev, i); if (IS_ERR(drv[i].base)) return PTR_ERR(drv[i].base); drv_dev = kzalloc(sizeof(*drv_dev), GFP_KERNEL); if (!drv_dev) return -ENOMEM; drv[i].id = i; drv[i].pdev = pdev; drv[i].dev = drv_dev; drv_dev->parent = &pdev->dev; drv_dev->of_node = np; dev_set_name(drv_dev, "%s:%pOFn%d", dev_name(drv_dev->parent), np, i); ret = device_register(drv_dev); if (ret) return ret; rsc_id = readl_relaxed(drv[i].base + RSC_DRV_ID); major_ver = rsc_id & (MAJOR_VER_MASK << MAJOR_VER_SHIFT); major_ver >>= MAJOR_VER_SHIFT; minor_ver = rsc_id & (MINOR_VER_MASK << MINOR_VER_SHIFT); minor_ver >>= MINOR_VER_SHIFT; if (major_ver >= 3) drv[i].regs = rpmh_rsc_reg_offset_ver_3_0; else drv[i].regs = rpmh_rsc_reg_offset_ver_2_7; ret = rpmh_probe_tcs_config(&drv[i]); if (ret) return ret; dev_set_drvdata(drv_dev, &drv[i]); drv[i].initialized = true; } for (i = 0; i < drv_count; i++) { if (!drv[i].initialized) continue; /* * CPU PM notification are not required for controllers that support * 'HW solver' mode where they can be in autonomous mode executing low * power mode to power down. */ solver_config = readl_relaxed(drv[i].base + drv[i].regs[DRV_SOLVER_CONFIG]); solver_config &= DRV_HW_SOLVER_MASK << DRV_HW_SOLVER_SHIFT; solver_config = solver_config >> DRV_HW_SOLVER_SHIFT; spin_lock_init(&drv[i].lock); spin_lock_init(&drv[i].client.cache_lock); if (of_find_property(dn, "power-domains", NULL) && pdev->dev.pm_domain) { ret = rpmh_rsc_pd_attach(&drv[i], &pdev->dev); if (ret) return ret; } else if (!solver_config && !of_find_property(dn, "qcom,hw-channel", NULL)) { drv[i].rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; cpu_pm_register_notifier(&drv[i].rsc_pm); } else if (solver_config) { drv[i].client.flags = SOLVER_PRESENT; } else { /* * The requets for HW channel TCSes has to be either * RPMH_SLEEP_STATE or RPMH_WAKE_ONLY_STATE. * * Assume 'solver' state which does nothing but to disallow * RPMH_ACTIVE_ONLY_STATE requests. */ drv[i].client.flags = SOLVER_PRESENT | HW_CHANNEL_PRESENT; drv[i].client.in_solver_mode = true; drv[i].in_solver_mode = true; drv[i].regs = rpmh_rsc_reg_offset_ver_3_0_hw_channel; } init_waitqueue_head(&drv[i].tcs_wait); bitmap_zero(drv[i].tcs_in_use, MAX_TCS_NR); drv[i].client.non_batch_cache = devm_kcalloc(&pdev->dev, CMD_DB_MAX_RESOURCES, sizeof(struct cache_req), GFP_KERNEL); if (!drv[i].client.non_batch_cache) return -ENOMEM; for (j = 0; j < CMD_DB_MAX_RESOURCES; j++) INIT_LIST_HEAD(&drv[i].client.non_batch_cache[j].list); irq = platform_get_irq(pdev, drv[i].id); if (irq < 0) return irq; drv[i].irq = irq; ret = devm_request_irq(&pdev->dev, irq, tcs_tx_done, IRQF_TRIGGER_HIGH | IRQF_NO_SUSPEND, drv[i].name, &drv[i]); if (ret) return ret; #if IS_ENABLED(CONFIG_IPC_LOGGING) drv[i].ipc_log_ctx = ipc_log_context_create( RSC_DRV_IPC_LOG_SIZE, drv[i].name, 0); #endif if (__rsc_count < MAX_RSC_COUNT) __rsc_drv[__rsc_count++] = &drv[i]; ret = devm_of_platform_populate(drv[i].dev); if (ret) return ret; } INIT_LIST_HEAD(&rsc_top->list); list_add_tail(&rsc_top->list, &rpmh_rsc_dev_list); dev_set_drvdata(&pdev->dev, rsc_top); return devm_of_platform_populate(&pdev->dev); } static const struct dev_pm_ops rpmh_rsc_dev_pm_ops = { .poweroff_noirq = rpmh_rsc_poweroff_noirq, .restore_noirq = rpmh_rsc_restore_noirq, }; static const struct of_device_id rpmh_drv_match[] = { { .compatible = "qcom,rpmh-rsc", }, { } }; MODULE_DEVICE_TABLE(of, rpmh_drv_match); static struct platform_driver rpmh_driver = { .probe = rpmh_rsc_probe, .driver = { .name = "rpmh", .of_match_table = rpmh_drv_match, .pm = &rpmh_rsc_dev_pm_ops, .suppress_bind_attrs = true, }, }; static int __init rpmh_driver_init(void) { return platform_driver_register(&rpmh_driver); } arch_initcall(rpmh_driver_init); MODULE_DESCRIPTION("Qualcomm Technologies, Inc. RPMh Driver"); MODULE_LICENSE("GPL v2");