// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2015, 2017-2018, 2022, The Linux Foundation. All rights reserved. * Copyright (c) 2024, Qualcomm Innovation Center, Inc. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "gdsc.h" #define PWR_ON_MASK BIT(31) #define EN_REST_WAIT_MASK GENMASK_ULL(23, 20) #define EN_FEW_WAIT_MASK GENMASK_ULL(19, 16) #define CLK_DIS_WAIT_MASK GENMASK_ULL(15, 12) #define SW_OVERRIDE_MASK BIT(2) #define HW_CONTROL_MASK BIT(1) #define SW_COLLAPSE_MASK BIT(0) #define GMEM_CLAMP_IO_MASK BIT(0) #define GMEM_RESET_MASK BIT(4) /* CFG_GDSCR */ #define GDSC_POWER_UP_COMPLETE BIT(16) #define GDSC_POWER_DOWN_COMPLETE BIT(15) #define GDSC_RETAIN_FF_ENABLE BIT(11) #define CFG_GDSCR_OFFSET 0x4 /* Wait 2^n CXO cycles between all states. Here, n=2 (4 cycles). */ #define EN_REST_WAIT_VAL 0x2 #define EN_FEW_WAIT_VAL 0x8 #define CLK_DIS_WAIT_VAL 0x2 /* Transition delay shifts */ #define EN_REST_WAIT_SHIFT 20 #define EN_FEW_WAIT_SHIFT 16 #define CLK_DIS_WAIT_SHIFT 12 #define RETAIN_MEM BIT(14) #define RETAIN_PERIPH BIT(13) #define STATUS_POLL_TIMEOUT_US 1500 #define TIMEOUT_US 500 #define domain_to_gdsc(domain) container_of(domain, struct gdsc, pd) enum gdsc_status { GDSC_OFF, GDSC_ON }; /* Returns 1 if GDSC status is status, 0 if not, and < 0 on error */ static int gdsc_check_status(struct gdsc *sc, enum gdsc_status status) { unsigned int reg; u32 val; int ret; if (sc->flags & POLL_CFG_GDSCR) reg = sc->gdscr + CFG_GDSCR_OFFSET; else if (sc->gds_hw_ctrl) reg = sc->gds_hw_ctrl; else reg = sc->gdscr; ret = regmap_read(sc->regmap, reg, &val); if (ret) return ret; if (sc->flags & POLL_CFG_GDSCR) { switch (status) { case GDSC_ON: return !!(val & GDSC_POWER_UP_COMPLETE); case GDSC_OFF: return !!(val & GDSC_POWER_DOWN_COMPLETE); } } switch (status) { case GDSC_ON: return !!(val & PWR_ON_MASK); case GDSC_OFF: return !(val & PWR_ON_MASK); } return -EINVAL; } static int gdsc_hwctrl(struct gdsc *sc, bool en) { u32 val = en ? HW_CONTROL_MASK : 0; return regmap_update_bits(sc->regmap, sc->gdscr, HW_CONTROL_MASK, val); } static int gdsc_poll_status(struct gdsc *sc, enum gdsc_status status) { ktime_t start; start = ktime_get(); do { if (gdsc_check_status(sc, status)) return 0; } while (ktime_us_delta(ktime_get(), start) < STATUS_POLL_TIMEOUT_US); if (gdsc_check_status(sc, status)) return 0; return -ETIMEDOUT; } static int gdsc_update_collapse_bit(struct gdsc *sc, bool val) { u32 reg, mask; int ret; if (sc->collapse_mask) { reg = sc->collapse_ctrl; mask = sc->collapse_mask; } else { reg = sc->gdscr; mask = SW_COLLAPSE_MASK; } ret = regmap_update_bits(sc->regmap, reg, mask, val ? mask : 0); if (ret) return ret; return 0; } static void log_gdsc_debug_regs(struct gdsc *sc) { u32 val; pr_err("Dumping %s registers\n", sc->pd.name); regmap_read(sc->regmap, sc->gds_hw_ctrl, &val); pr_err("HW_CTRL_CFG1_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gdscr, &val); pr_err("GDSCR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gdscr + 4, &val); pr_err("CFG_GDSCR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gdscr + 8, &val); pr_err("CFG2_GDSCR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gdscr + 0xc, &val); pr_err("CFG3_GDSCR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gdscr + 0x10, &val); pr_err("CFG4_GDSCR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl, &val); pr_err("HW_CTRL_CFG1_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl + 4, &val); pr_err("HW_CTRL_CFG2_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl + 8, &val); pr_err("HW_CTRL_DVM_STATUS_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl + 0xc, &val); pr_err("HW_CTRL_HALT1_STATUS_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl + 0x10, &val); pr_err("HW_CTRL_HALT2_STATUS_GDSR : 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl + 0x14, &val); pr_err("HW_CTRL_REQ_SW_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl + 0x18, &val); pr_err("HW_CTRL_IRQ_STATUS_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl + 0x20, &val); pr_err("HW_CTRL_IRQ_CLEAR_GDSR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gds_hw_ctrl, &val); pr_err("HW_CTRL_CFG1_GDSR: 0x%.8x\n\n", val); } static int gdsc_toggle_logic(struct gdsc *sc, enum gdsc_status status, bool wait) { int ret; u32 val; if (status == GDSC_ON && sc->rsupply) { ret = regulator_enable(sc->rsupply); if (ret < 0) return ret; } regmap_read(sc->regmap, sc->gdscr, &val); if (val & HW_CONTROL_MASK) { pr_debug("%s in HW control mode\n", sc->pd.name); return 0; } ret = gdsc_update_collapse_bit(sc, status == GDSC_OFF); /* If disabling votable gdscs, don't poll on status */ if ((sc->flags & VOTABLE) && status == GDSC_OFF && !wait) { /* * Add a short delay here to ensure that an enable * right after it was disabled does not put it in an * unknown state */ udelay(TIMEOUT_US); goto out; } if (sc->gds_hw_ctrl) { /* * The gds hw controller asserts/de-asserts the status bit soon * after it receives a power on/off request from a master. * The controller then takes around 8 xo cycles to start its * internal state machine and update the status bit. During * this time, the status bit does not reflect the true status * of the core. * Add a delay of 1 us between writing to the SW_COLLAPSE bit * and polling the status bit. */ udelay(1); } ret = gdsc_poll_status(sc, status); if (ret && sc->gds_hw_ctrl) { pr_err("%s enable timed out, Re-polling\n", sc->pd.name); log_gdsc_debug_regs(sc); ret = gdsc_poll_status(sc, status); if (ret) { log_gdsc_debug_regs(sc); udelay(500); log_gdsc_debug_regs(sc); udelay(1000); log_gdsc_debug_regs(sc); } } WARN(ret, "%s status stuck at 'o%s'", sc->pd.name, status ? "ff" : "n"); out: if (!ret && status == GDSC_OFF && sc->rsupply) { ret = regulator_disable(sc->rsupply); if (ret < 0) return ret; } return ret; } static inline int gdsc_deassert_reset(struct gdsc *sc) { int i; for (i = 0; i < sc->reset_count; i++) sc->rcdev->ops->deassert(sc->rcdev, sc->resets[i]); return 0; } static inline int gdsc_assert_reset(struct gdsc *sc) { int i; for (i = 0; i < sc->reset_count; i++) sc->rcdev->ops->assert(sc->rcdev, sc->resets[i]); return 0; } static inline void gdsc_force_mem_on(struct gdsc *sc) { int i; u32 mask = RETAIN_MEM; if (!(sc->flags & NO_RET_PERIPH)) mask |= RETAIN_PERIPH; for (i = 0; i < sc->cxc_count; i++) regmap_update_bits(sc->regmap, sc->cxcs[i], mask, mask); } static inline void gdsc_clear_mem_on(struct gdsc *sc) { int i; u32 mask = RETAIN_MEM; if (!(sc->flags & NO_RET_PERIPH)) mask |= RETAIN_PERIPH; for (i = 0; i < sc->cxc_count; i++) regmap_update_bits(sc->regmap, sc->cxcs[i], mask, 0); } static inline void gdsc_deassert_clamp_io(struct gdsc *sc) { regmap_update_bits(sc->regmap, sc->clamp_io_ctrl, GMEM_CLAMP_IO_MASK, 0); } static inline void gdsc_assert_clamp_io(struct gdsc *sc) { regmap_update_bits(sc->regmap, sc->clamp_io_ctrl, GMEM_CLAMP_IO_MASK, 1); } static inline void gdsc_assert_reset_aon(struct gdsc *sc) { regmap_update_bits(sc->regmap, sc->clamp_io_ctrl, GMEM_RESET_MASK, 1); udelay(1); regmap_update_bits(sc->regmap, sc->clamp_io_ctrl, GMEM_RESET_MASK, 0); } static void gdsc_retain_ff_on(struct gdsc *sc) { u32 mask = GDSC_RETAIN_FF_ENABLE; regmap_update_bits(sc->regmap, sc->gdscr, mask, mask); } static int gdsc_enable(struct generic_pm_domain *domain) { struct gdsc *sc = domain_to_gdsc(domain); int ret; if (sc->pwrsts == PWRSTS_ON) return gdsc_deassert_reset(sc); if (sc->flags & SW_RESET) { gdsc_assert_reset(sc); udelay(1); gdsc_deassert_reset(sc); } if (sc->flags & CLAMP_IO) { if (sc->flags & AON_RESET) gdsc_assert_reset_aon(sc); gdsc_deassert_clamp_io(sc); } ret = gdsc_toggle_logic(sc, GDSC_ON, false); if (ret) return ret; if (sc->pwrsts & PWRSTS_OFF) gdsc_force_mem_on(sc); /* * If clocks to this power domain were already on, they will take an * additional 4 clock cycles to re-enable after the power domain is * enabled. Delay to account for this. A delay is also needed to ensure * clocks are not enabled within 400ns of enabling power to the * memories. */ udelay(1); /* Turn on HW trigger mode if supported */ if (sc->flags & HW_CTRL) { ret = gdsc_hwctrl(sc, true); if (ret) return ret; /* * Wait for the GDSC to go through a power down and * up cycle. In case a firmware ends up polling status * bits for the gdsc, it might read an 'on' status before * the GDSC can finish the power cycle. * We wait 1us before returning to ensure the firmware * can't immediately poll the status bits. */ udelay(1); } if (sc->flags & RETAIN_FF_ENABLE) gdsc_retain_ff_on(sc); return 0; } static int gdsc_disable(struct generic_pm_domain *domain) { struct gdsc *sc = domain_to_gdsc(domain); int ret; if (sc->pwrsts == PWRSTS_ON) return gdsc_assert_reset(sc); /* Turn off HW trigger mode if supported */ if (sc->flags & HW_CTRL) { if (sc->flags & HW_CTRL_SKIP_DIS) { if (sc->rsupply) return regulator_disable(sc->rsupply); return 0; } ret = gdsc_hwctrl(sc, false); if (ret < 0) return ret; /* * Wait for the GDSC to go through a power down and * up cycle. In case we end up polling status * bits for the gdsc before the power cycle is completed * it might read an 'on' status wrongly. */ udelay(1); ret = gdsc_poll_status(sc, GDSC_ON); if (ret) return ret; } if (sc->pwrsts & PWRSTS_OFF) gdsc_clear_mem_on(sc); /* * If the GDSC supports only a Retention state, apart from ON, * leave it in ON state. * There is no SW control to transition the GDSC into * Retention state. This happens in HW when the parent * domain goes down to a Low power state */ if (sc->pwrsts == PWRSTS_RET_ON) return 0; ret = gdsc_toggle_logic(sc, GDSC_OFF, domain->synced_poweroff); if (ret) return ret; if (sc->flags & CLAMP_IO) gdsc_assert_clamp_io(sc); return 0; } static int gdsc_init(struct gdsc *sc) { u32 mask, val; int on, ret; /* * Disable HW trigger: collapse/restore occur based on registers writes. * Disable SW override: Use hardware state-machine for sequencing. * Configure wait time between states. */ mask = HW_CONTROL_MASK | SW_OVERRIDE_MASK | EN_REST_WAIT_MASK | EN_FEW_WAIT_MASK | CLK_DIS_WAIT_MASK; if (!sc->en_rest_wait_val) sc->en_rest_wait_val = EN_REST_WAIT_VAL; if (!sc->en_few_wait_val) sc->en_few_wait_val = EN_FEW_WAIT_VAL; if (!sc->clk_dis_wait_val) sc->clk_dis_wait_val = CLK_DIS_WAIT_VAL; val = sc->en_rest_wait_val << EN_REST_WAIT_SHIFT | sc->en_few_wait_val << EN_FEW_WAIT_SHIFT | sc->clk_dis_wait_val << CLK_DIS_WAIT_SHIFT; ret = regmap_update_bits(sc->regmap, sc->gdscr, mask, val); if (ret) return ret; /* Force gdsc ON if only ON state is supported */ if (sc->pwrsts == PWRSTS_ON) { ret = gdsc_toggle_logic(sc, GDSC_ON, false); if (ret) return ret; } on = gdsc_check_status(sc, GDSC_ON); if (on < 0) return on; if (on) { /* The regulator must be on, sync the kernel state */ if (sc->rsupply) { ret = regulator_enable(sc->rsupply); if (ret < 0) return ret; } /* * Votable GDSCs can be ON due to Vote from other masters. * If a Votable GDSC is ON, make sure we have a Vote. */ if (sc->flags & VOTABLE) { ret = gdsc_update_collapse_bit(sc, false); if (ret) goto err_disable_supply; } /* Turn on HW trigger mode if supported */ if (sc->flags & HW_CTRL) { ret = gdsc_hwctrl(sc, true); if (ret < 0) goto err_disable_supply; } /* * Make sure the retain bit is set if the GDSC is already on, * otherwise we end up turning off the GDSC and destroying all * the register contents that we thought we were saving. */ if (sc->flags & RETAIN_FF_ENABLE) gdsc_retain_ff_on(sc); } else if (sc->flags & ALWAYS_ON) { /* If ALWAYS_ON GDSCs are not ON, turn them ON */ gdsc_enable(&sc->pd); on = true; } if (on || (sc->pwrsts & PWRSTS_RET)) gdsc_force_mem_on(sc); else gdsc_clear_mem_on(sc); if (sc->flags & ALWAYS_ON) sc->pd.flags |= GENPD_FLAG_ALWAYS_ON; if (!sc->pd.power_off) sc->pd.power_off = gdsc_disable; if (!sc->pd.power_on) sc->pd.power_on = gdsc_enable; ret = pm_genpd_init(&sc->pd, NULL, !on); if (ret) goto err_disable_supply; return 0; err_disable_supply: if (on && sc->rsupply) regulator_disable(sc->rsupply); return ret; } int gdsc_register(struct gdsc_desc *desc, struct reset_controller_dev *rcdev, struct regmap *regmap) { int i, ret; struct genpd_onecell_data *data; struct device *dev = desc->dev; struct gdsc **scs = desc->scs; size_t num = desc->num; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; data->domains = devm_kcalloc(dev, num, sizeof(*data->domains), GFP_KERNEL); if (!data->domains) return -ENOMEM; for (i = 0; i < num; i++) { if (!scs[i] || !scs[i]->supply) continue; scs[i]->rsupply = devm_regulator_get(dev, scs[i]->supply); if (IS_ERR(scs[i]->rsupply)) return PTR_ERR(scs[i]->rsupply); } data->num_domains = num; for (i = 0; i < num; i++) { if (!scs[i]) continue; scs[i]->regmap = regmap; scs[i]->rcdev = rcdev; ret = gdsc_init(scs[i]); if (ret) return ret; data->domains[i] = &scs[i]->pd; } /* Add subdomains */ for (i = 0; i < num; i++) { if (!scs[i]) continue; if (scs[i]->parent) pm_genpd_add_subdomain(scs[i]->parent, &scs[i]->pd); else if (!IS_ERR_OR_NULL(dev->pm_domain)) pm_genpd_add_subdomain(pd_to_genpd(dev->pm_domain), &scs[i]->pd); } return of_genpd_add_provider_onecell(dev->of_node, data); } void gdsc_unregister(struct gdsc_desc *desc) { int i; struct device *dev = desc->dev; struct gdsc **scs = desc->scs; size_t num = desc->num; /* Remove subdomains */ for (i = 0; i < num; i++) { if (!scs[i]) continue; if (scs[i]->parent) pm_genpd_remove_subdomain(scs[i]->parent, &scs[i]->pd); else if (!IS_ERR_OR_NULL(dev->pm_domain)) pm_genpd_remove_subdomain(pd_to_genpd(dev->pm_domain), &scs[i]->pd); } of_genpd_del_provider(dev->of_node); } /* * On SDM845+ the GPU GX domain is *almost* entirely controlled by the GMU * running in the CX domain so the CPU doesn't need to know anything about the * GX domain EXCEPT.... * * Hardware constraints dictate that the GX be powered down before the CX. If * the GMU crashes it could leave the GX on. In order to successfully bring back * the device the CPU needs to disable the GX headswitch. There being no sane * way to reach in and touch that register from deep inside the GPU driver we * need to set up the infrastructure to be able to ensure that the GPU can * ensure that the GX is off during this super special case. We do this by * defining a GX gdsc with a dummy enable function and a "default" disable * function. * * This allows us to attach with genpd_dev_pm_attach_by_name() in the GPU * driver. During power up, nothing will happen from the CPU (and the GMU will * power up normally but during power down this will ensure that the GX domain * is *really* off - this gives us a semi standard way of doing what we need. */ int gdsc_gx_do_nothing_enable(struct generic_pm_domain *domain) { struct gdsc *sc = domain_to_gdsc(domain); int ret = 0; /* Enable the parent supply, when controlled through the regulator framework. */ if (sc->rsupply) ret = regulator_enable(sc->rsupply); /* Do nothing with the GDSC itself */ return ret; } EXPORT_SYMBOL_GPL(gdsc_gx_do_nothing_enable); static void gdsc_debug_print_registers(struct gdsc *sc) { u32 val; regmap_read(sc->regmap, sc->gdscr, &val); pr_err("GDSCR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gdscr + 4, &val); pr_err("CFG_GDSCR: 0x%.8x\n", val); regmap_read(sc->regmap, sc->gdscr + 8, &val); pr_err("CFG2_GDSCR: 0x%.8x\n", val); if (sc->gds_hw_ctrl) { regmap_read(sc->regmap, sc->gds_hw_ctrl, &val); pr_err("GDS_HW_CTRL: 0x%.8x\n", val); } if (sc->collapse_ctrl) { regmap_read(sc->regmap, sc->collapse_ctrl, &val); pr_err("COLLAPSE_CTRL: 0x%.8x\n", val); } } static void genpd_dump_consumers(struct generic_pm_domain *genpd) { static const char * const status_lookup[] = { [RPM_ACTIVE] = "active", [RPM_RESUMING] = "resuming", [RPM_SUSPENDED] = "suspended", [RPM_SUSPENDING] = "suspending" }; struct pm_domain_data *pm_data; const char *kobj_path; const char *p = ""; struct device *dev; pr_err("%-30s %-50s\n", genpd->name, genpd->status ? "off" : "on"); pr_err("Consumers:\n"); list_for_each_entry(pm_data, &genpd->dev_list, list_node) { kobj_path = kobject_get_path(&pm_data->dev->kobj, genpd->flags & GENPD_FLAG_IRQ_SAFE ? GFP_ATOMIC : GFP_KERNEL); if (kobj_path == NULL) continue; dev = pm_data->dev; if (dev->power.runtime_error) p = "error"; else if (dev->power.disable_depth) p = "unsupported"; else if (dev->power.runtime_status < ARRAY_SIZE(status_lookup)) p = status_lookup[dev->power.runtime_status]; pr_err("%-50s %-25s\n", kobj_path, p); pr_err("usage_count:%d\n", atomic_read(&dev->power.usage_count)); pr_err("last_busy:%llu\n", dev->power.last_busy); pr_err("timer_expires:%llu\n", dev->power.timer_expires); pr_err("timer_state:%d\n", dev->power.suspend_timer.state); pr_err("rpm_request:%d\n", dev->power.request); pr_err("request_pending:%u\n", dev->power.request_pending); pr_err("timer_autosuspends:%d\n", dev->power.timer_autosuspends); pr_err("runtime_error:%d\n", dev->power.runtime_error); pr_err("autosuspend_delay:%d\n", dev->power.autosuspend_delay); pr_err("disable_depth:%u\n", dev->power.disable_depth); pr_err("child_count:%d\n", atomic_read(&dev->power.child_count)); pr_err("deferred_resume:%u\n", dev->power.deferred_resume); pr_err("ignore_children:%u\n", dev->power.ignore_children); pr_err("use_autosuspend:%u\n", dev->power.use_autosuspend); pr_err("current_time: %llu\n", ktime_get_mono_fast_ns()); kfree(kobj_path); } pr_err("device_count: %d\n", genpd->device_count); pr_err("prepared_count: %d\n", genpd->prepared_count); pr_err("suspended_count: %d\n", genpd->suspended_count); pr_err("sd_count: %d\n", atomic_read(&genpd->sd_count)); } void qcom_gdsc_pd_dump(struct device *device) { struct generic_pm_domain *genpd, *child; struct gpd_link *link; struct gdsc *sc; if (!device) { pr_err("Null device handle passed\n"); return; } if (device && IS_ERR_OR_NULL(device->pm_domain)) { pr_err("No pm_domain linked to device\n"); return; } genpd = pd_to_genpd(device->pm_domain); if (IS_ERR(genpd)) { pr_err("No pd linked to device\n"); return; } if (mutex_lock_interruptible(&genpd->mlock)) { pr_err("Failed to acquire %s genpd lock\n", genpd->name); return; } genpd_dump_consumers(genpd); pr_err("Child_domains:\n"); list_for_each_entry(link, &genpd->parent_links, parent_node) { child = link->child; if (mutex_lock_interruptible(&child->mlock)) { pr_err("Failed to acquire %s genpd lock\n", child->name); mutex_unlock(&genpd->mlock); return; } genpd_dump_consumers(child); mutex_unlock(&child->mlock); } sc = domain_to_gdsc(genpd); pr_err("Dumping %s registers\n", sc->pd.name); gdsc_debug_print_registers(sc); mutex_unlock(&genpd->mlock); } EXPORT_SYMBOL_GPL(qcom_gdsc_pd_dump);