blob: f752653ccb47e2b637f5a893cdad40562d759636 [file] [log] [blame]
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2020 Intel Corporation */
#include <linux/iopoll.h>
#include "adf_accel_devices.h"
#include "adf_cfg_services.h"
#include "adf_common_drv.h"
#include "adf_gen4_hw_data.h"
#include "adf_gen4_pm.h"
static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size)
{
return BUILD_RING_BASE_ADDR(addr, size);
}
static u32 read_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring)
{
return READ_CSR_RING_HEAD(csr_base_addr, bank, ring);
}
static void write_csr_ring_head(void __iomem *csr_base_addr, u32 bank, u32 ring,
u32 value)
{
WRITE_CSR_RING_HEAD(csr_base_addr, bank, ring, value);
}
static u32 read_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring)
{
return READ_CSR_RING_TAIL(csr_base_addr, bank, ring);
}
static void write_csr_ring_tail(void __iomem *csr_base_addr, u32 bank, u32 ring,
u32 value)
{
WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value);
}
static u32 read_csr_e_stat(void __iomem *csr_base_addr, u32 bank)
{
return READ_CSR_E_STAT(csr_base_addr, bank);
}
static void write_csr_ring_config(void __iomem *csr_base_addr, u32 bank, u32 ring,
u32 value)
{
WRITE_CSR_RING_CONFIG(csr_base_addr, bank, ring, value);
}
static void write_csr_ring_base(void __iomem *csr_base_addr, u32 bank, u32 ring,
dma_addr_t addr)
{
WRITE_CSR_RING_BASE(csr_base_addr, bank, ring, addr);
}
static void write_csr_int_flag(void __iomem *csr_base_addr, u32 bank,
u32 value)
{
WRITE_CSR_INT_FLAG(csr_base_addr, bank, value);
}
static void write_csr_int_srcsel(void __iomem *csr_base_addr, u32 bank)
{
WRITE_CSR_INT_SRCSEL(csr_base_addr, bank);
}
static void write_csr_int_col_en(void __iomem *csr_base_addr, u32 bank, u32 value)
{
WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value);
}
static void write_csr_int_col_ctl(void __iomem *csr_base_addr, u32 bank,
u32 value)
{
WRITE_CSR_INT_COL_CTL(csr_base_addr, bank, value);
}
static void write_csr_int_flag_and_col(void __iomem *csr_base_addr, u32 bank,
u32 value)
{
WRITE_CSR_INT_FLAG_AND_COL(csr_base_addr, bank, value);
}
static void write_csr_ring_srv_arb_en(void __iomem *csr_base_addr, u32 bank,
u32 value)
{
WRITE_CSR_RING_SRV_ARB_EN(csr_base_addr, bank, value);
}
void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops)
{
csr_ops->build_csr_ring_base_addr = build_csr_ring_base_addr;
csr_ops->read_csr_ring_head = read_csr_ring_head;
csr_ops->write_csr_ring_head = write_csr_ring_head;
csr_ops->read_csr_ring_tail = read_csr_ring_tail;
csr_ops->write_csr_ring_tail = write_csr_ring_tail;
csr_ops->read_csr_e_stat = read_csr_e_stat;
csr_ops->write_csr_ring_config = write_csr_ring_config;
csr_ops->write_csr_ring_base = write_csr_ring_base;
csr_ops->write_csr_int_flag = write_csr_int_flag;
csr_ops->write_csr_int_srcsel = write_csr_int_srcsel;
csr_ops->write_csr_int_col_en = write_csr_int_col_en;
csr_ops->write_csr_int_col_ctl = write_csr_int_col_ctl;
csr_ops->write_csr_int_flag_and_col = write_csr_int_flag_and_col;
csr_ops->write_csr_ring_srv_arb_en = write_csr_ring_srv_arb_en;
}
EXPORT_SYMBOL_GPL(adf_gen4_init_hw_csr_ops);
u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self)
{
return ADF_GEN4_ACCELERATORS_MASK;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_accel_mask);
u32 adf_gen4_get_num_accels(struct adf_hw_device_data *self)
{
return ADF_GEN4_MAX_ACCELERATORS;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_num_accels);
u32 adf_gen4_get_num_aes(struct adf_hw_device_data *self)
{
if (!self || !self->ae_mask)
return 0;
return hweight32(self->ae_mask);
}
EXPORT_SYMBOL_GPL(adf_gen4_get_num_aes);
u32 adf_gen4_get_misc_bar_id(struct adf_hw_device_data *self)
{
return ADF_GEN4_PMISC_BAR;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_misc_bar_id);
u32 adf_gen4_get_etr_bar_id(struct adf_hw_device_data *self)
{
return ADF_GEN4_ETR_BAR;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_etr_bar_id);
u32 adf_gen4_get_sram_bar_id(struct adf_hw_device_data *self)
{
return ADF_GEN4_SRAM_BAR;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_sram_bar_id);
enum dev_sku_info adf_gen4_get_sku(struct adf_hw_device_data *self)
{
return DEV_SKU_1;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_sku);
void adf_gen4_get_arb_info(struct arb_info *arb_info)
{
arb_info->arb_cfg = ADF_GEN4_ARB_CONFIG;
arb_info->arb_offset = ADF_GEN4_ARB_OFFSET;
arb_info->wt2sam_offset = ADF_GEN4_ARB_WRK_2_SER_MAP_OFFSET;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_arb_info);
void adf_gen4_get_admin_info(struct admin_info *admin_csrs_info)
{
admin_csrs_info->mailbox_offset = ADF_GEN4_MAILBOX_BASE_OFFSET;
admin_csrs_info->admin_msg_ur = ADF_GEN4_ADMINMSGUR_OFFSET;
admin_csrs_info->admin_msg_lr = ADF_GEN4_ADMINMSGLR_OFFSET;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_admin_info);
u32 adf_gen4_get_heartbeat_clock(struct adf_hw_device_data *self)
{
/*
* GEN4 uses KPT counter for HB
*/
return ADF_GEN4_KPT_COUNTER_FREQ;
}
EXPORT_SYMBOL_GPL(adf_gen4_get_heartbeat_clock);
void adf_gen4_enable_error_correction(struct adf_accel_dev *accel_dev)
{
struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR];
void __iomem *csr = misc_bar->virt_addr;
/* Enable all in errsou3 except VFLR notification on host */
ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, ADF_GEN4_VFLNOTIFY);
}
EXPORT_SYMBOL_GPL(adf_gen4_enable_error_correction);
void adf_gen4_enable_ints(struct adf_accel_dev *accel_dev)
{
void __iomem *addr;
addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr;
/* Enable bundle interrupts */
ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X0_MASK_OFFSET, 0);
ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X1_MASK_OFFSET, 0);
/* Enable misc interrupts */
ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_MASK_OFFSET, 0);
}
EXPORT_SYMBOL_GPL(adf_gen4_enable_ints);
int adf_gen4_init_device(struct adf_accel_dev *accel_dev)
{
void __iomem *addr;
u32 status;
u32 csr;
int ret;
addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr;
/* Temporarily mask PM interrupt */
csr = ADF_CSR_RD(addr, ADF_GEN4_ERRMSK2);
csr |= ADF_GEN4_PM_SOU;
ADF_CSR_WR(addr, ADF_GEN4_ERRMSK2, csr);
/* Set DRV_ACTIVE bit to power up the device */
ADF_CSR_WR(addr, ADF_GEN4_PM_INTERRUPT, ADF_GEN4_PM_DRV_ACTIVE);
/* Poll status register to make sure the device is powered up */
ret = read_poll_timeout(ADF_CSR_RD, status,
status & ADF_GEN4_PM_INIT_STATE,
ADF_GEN4_PM_POLL_DELAY_US,
ADF_GEN4_PM_POLL_TIMEOUT_US, true, addr,
ADF_GEN4_PM_STATUS);
if (ret)
dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n");
return ret;
}
EXPORT_SYMBOL_GPL(adf_gen4_init_device);
static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper,
u32 *lower)
{
*lower = lower_32_bits(value);
*upper = upper_32_bits(value);
}
void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
u32 ssm_wdt_pke_high = 0;
u32 ssm_wdt_pke_low = 0;
u32 ssm_wdt_high = 0;
u32 ssm_wdt_low = 0;
/* Convert 64bit WDT timer value into 32bit values for
* mmio write to 32bit CSRs.
*/
adf_gen4_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low);
adf_gen4_unpack_ssm_wdtimer(timer_val_pke, &ssm_wdt_pke_high,
&ssm_wdt_pke_low);
/* Enable WDT for sym and dc */
ADF_CSR_WR(pmisc_addr, ADF_SSMWDTL_OFFSET, ssm_wdt_low);
ADF_CSR_WR(pmisc_addr, ADF_SSMWDTH_OFFSET, ssm_wdt_high);
/* Enable WDT for pke */
ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ssm_wdt_pke_low);
ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high);
}
EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer);
/*
* The vector routing table is used to select the MSI-X entry to use for each
* interrupt source.
* The first ADF_GEN4_ETR_MAX_BANKS entries correspond to ring interrupts.
* The final entry corresponds to VF2PF or error interrupts.
* This vector table could be used to configure one MSI-X entry to be shared
* between multiple interrupt sources.
*
* The default routing is set to have a one to one correspondence between the
* interrupt source and the MSI-X entry used.
*/
void adf_gen4_set_msix_default_rttable(struct adf_accel_dev *accel_dev)
{
void __iomem *csr;
int i;
csr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr;
for (i = 0; i <= ADF_GEN4_ETR_MAX_BANKS; i++)
ADF_CSR_WR(csr, ADF_GEN4_MSIX_RTTABLE_OFFSET(i), i);
}
EXPORT_SYMBOL_GPL(adf_gen4_set_msix_default_rttable);
int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
{
return 0;
}
EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled);
static int reset_ring_pair(void __iomem *csr, u32 bank_number)
{
u32 status;
int ret;
/* Write rpresetctl register BIT(0) as 1
* Since rpresetctl registers have no RW fields, no need to preserve
* values for other bits. Just write directly.
*/
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
ADF_WQM_CSR_RPRESETCTL_RESET);
/* Read rpresetsts register and wait for rp reset to complete */
ret = read_poll_timeout(ADF_CSR_RD, status,
status & ADF_WQM_CSR_RPRESETSTS_STATUS,
ADF_RPRESET_POLL_DELAY_US,
ADF_RPRESET_POLL_TIMEOUT_US, true,
csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
if (!ret) {
/* When rp reset is done, clear rpresetsts */
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number),
ADF_WQM_CSR_RPRESETSTS_STATUS);
}
return ret;
}
int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
u32 etr_bar_id = hw_data->get_etr_bar_id(hw_data);
void __iomem *csr;
int ret;
if (bank_number >= hw_data->num_banks)
return -EINVAL;
dev_dbg(&GET_DEV(accel_dev),
"ring pair reset for bank:%d\n", bank_number);
csr = (&GET_BARS(accel_dev)[etr_bar_id])->virt_addr;
ret = reset_ring_pair(csr, bank_number);
if (ret)
dev_err(&GET_DEV(accel_dev),
"ring pair reset failed (timeout)\n");
else
dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");
return ret;
}
EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset);
static const u32 thrd_to_arb_map_dcc[] = {
0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x0000FFFF, 0x0000FFFF, 0x0000FFFF, 0x0000FFFF,
0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x0
};
static const u16 rp_group_to_arb_mask[] = {
[RP_GROUP_0] = 0x5,
[RP_GROUP_1] = 0xA,
};
static bool is_single_service(int service_id)
{
switch (service_id) {
case SVC_DC:
case SVC_SYM:
case SVC_ASYM:
return true;
case SVC_CY:
case SVC_CY2:
case SVC_DCC:
case SVC_ASYM_DC:
case SVC_DC_ASYM:
case SVC_SYM_DC:
case SVC_DC_SYM:
default:
return false;
}
}
int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
u32 *thd2arb_map = hw_data->thd_to_arb_map;
unsigned int ae_cnt, worker_obj_cnt, i, j;
unsigned long ae_mask, thds_mask;
int srv_id, rp_group;
u32 thd2arb_map_base;
u16 arb_mask;
if (!hw_data->get_rp_group || !hw_data->get_ena_thd_mask ||
!hw_data->get_num_aes || !hw_data->uof_get_num_objs ||
!hw_data->uof_get_ae_mask)
return -EFAULT;
srv_id = adf_get_service_enabled(accel_dev);
if (srv_id < 0)
return srv_id;
ae_cnt = hw_data->get_num_aes(hw_data);
worker_obj_cnt = hw_data->uof_get_num_objs(accel_dev) -
ADF_GEN4_ADMIN_ACCELENGINES;
if (srv_id == SVC_DCC) {
if (ae_cnt > ICP_QAT_HW_AE_DELIMITER)
return -EINVAL;
memcpy(thd2arb_map, thrd_to_arb_map_dcc,
array_size(sizeof(*thd2arb_map), ae_cnt));
return 0;
}
for (i = 0; i < worker_obj_cnt; i++) {
ae_mask = hw_data->uof_get_ae_mask(accel_dev, i);
rp_group = hw_data->get_rp_group(accel_dev, ae_mask);
thds_mask = hw_data->get_ena_thd_mask(accel_dev, i);
thd2arb_map_base = 0;
if (rp_group >= RP_GROUP_COUNT || rp_group < RP_GROUP_0)
return -EINVAL;
if (thds_mask == ADF_GEN4_ENA_THD_MASK_ERROR)
return -EINVAL;
if (is_single_service(srv_id))
arb_mask = rp_group_to_arb_mask[RP_GROUP_0] |
rp_group_to_arb_mask[RP_GROUP_1];
else
arb_mask = rp_group_to_arb_mask[rp_group];
for_each_set_bit(j, &thds_mask, ADF_NUM_THREADS_PER_AE)
thd2arb_map_base |= arb_mask << (j * 4);
for_each_set_bit(j, &ae_mask, ae_cnt)
thd2arb_map[j] = thd2arb_map_base;
}
return 0;
}
EXPORT_SYMBOL_GPL(adf_gen4_init_thd2arb_map);