blob: 4f5ca5ab13a40640578aed38f6ef1eea59609f56 [file] [log] [blame]
// SPDX-License-Identifier: GPL-2.0
/* Marvell RVU Admin Function driver
*
* Copyright (C) 2018 Marvell.
*
*/
#include <linux/bitfield.h>
#include <linux/module.h>
#include <linux/pci.h>
#include "rvu_struct.h"
#include "rvu_reg.h"
#include "rvu.h"
static int npa_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
struct npa_aq_inst_s *inst)
{
struct admin_queue *aq = block->aq;
struct npa_aq_res_s *result;
int timeout = 1000;
u64 reg, head;
result = (struct npa_aq_res_s *)aq->res->base;
/* Get current head pointer where to append this instruction */
reg = rvu_read64(rvu, block->addr, NPA_AF_AQ_STATUS);
head = (reg >> 4) & AQ_PTR_MASK;
memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
(void *)inst, aq->inst->entry_sz);
memset(result, 0, sizeof(*result));
/* sync into memory */
wmb();
/* Ring the doorbell and wait for result */
rvu_write64(rvu, block->addr, NPA_AF_AQ_DOOR, 1);
while (result->compcode == NPA_AQ_COMP_NOTDONE) {
cpu_relax();
udelay(1);
timeout--;
if (!timeout)
return -EBUSY;
}
if (result->compcode != NPA_AQ_COMP_GOOD) {
/* TODO: Replace this with some error code */
if (result->compcode == NPA_AQ_COMP_CTX_FAULT ||
result->compcode == NPA_AQ_COMP_LOCKERR ||
result->compcode == NPA_AQ_COMP_CTX_POISON) {
if (rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NPA0))
dev_err(rvu->dev,
"%s: Not able to unlock cachelines\n", __func__);
}
return -EBUSY;
}
return 0;
}
int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req,
struct npa_aq_enq_rsp *rsp)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
int blkaddr, npalf, rc = 0;
struct npa_aq_inst_s inst;
struct rvu_block *block;
struct admin_queue *aq;
struct rvu_pfvf *pfvf;
void *ctx, *mask;
bool ena;
pfvf = rvu_get_pfvf(rvu, pcifunc);
if (!pfvf->aura_ctx || req->aura_id >= pfvf->aura_ctx->qsize)
return NPA_AF_ERR_AQ_ENQUEUE;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
if (!pfvf->npalf || blkaddr < 0)
return NPA_AF_ERR_AF_LF_INVALID;
block = &hw->block[blkaddr];
aq = block->aq;
if (!aq) {
dev_warn(rvu->dev, "%s: NPA AQ not initialized\n", __func__);
return NPA_AF_ERR_AQ_ENQUEUE;
}
npalf = rvu_get_lf(rvu, block, pcifunc, 0);
if (npalf < 0)
return NPA_AF_ERR_AF_LF_INVALID;
memset(&inst, 0, sizeof(struct npa_aq_inst_s));
inst.cindex = req->aura_id;
inst.lf = npalf;
inst.ctype = req->ctype;
inst.op = req->op;
/* Currently we are not supporting enqueuing multiple instructions,
* so always choose first entry in result memory.
*/
inst.res_addr = (u64)aq->res->iova;
/* Hardware uses same aq->res->base for updating result of
* previous instruction hence wait here till it is done.
*/
spin_lock(&aq->lock);
/* Clean result + context memory */
memset(aq->res->base, 0, aq->res->entry_sz);
/* Context needs to be written at RES_ADDR + 128 */
ctx = aq->res->base + 128;
/* Mask needs to be written at RES_ADDR + 256 */
mask = aq->res->base + 256;
switch (req->op) {
case NPA_AQ_INSTOP_WRITE:
/* Copy context and write mask */
if (req->ctype == NPA_AQ_CTYPE_AURA) {
memcpy(mask, &req->aura_mask,
sizeof(struct npa_aura_s));
memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
} else {
memcpy(mask, &req->pool_mask,
sizeof(struct npa_pool_s));
memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
}
break;
case NPA_AQ_INSTOP_INIT:
if (req->ctype == NPA_AQ_CTYPE_AURA) {
if (req->aura.pool_addr >= pfvf->pool_ctx->qsize) {
rc = NPA_AF_ERR_AQ_FULL;
break;
}
/* Set pool's context address */
req->aura.pool_addr = pfvf->pool_ctx->iova +
(req->aura.pool_addr * pfvf->pool_ctx->entry_sz);
memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
} else { /* POOL's context */
memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
}
break;
case NPA_AQ_INSTOP_NOP:
case NPA_AQ_INSTOP_READ:
case NPA_AQ_INSTOP_LOCK:
case NPA_AQ_INSTOP_UNLOCK:
break;
default:
rc = NPA_AF_ERR_AQ_FULL;
break;
}
if (rc) {
spin_unlock(&aq->lock);
return rc;
}
/* Submit the instruction to AQ */
rc = npa_aq_enqueue_wait(rvu, block, &inst);
if (rc) {
spin_unlock(&aq->lock);
return rc;
}
/* Set aura bitmap if aura hw context is enabled */
if (req->ctype == NPA_AQ_CTYPE_AURA) {
if (req->op == NPA_AQ_INSTOP_INIT && req->aura.ena)
__set_bit(req->aura_id, pfvf->aura_bmap);
if (req->op == NPA_AQ_INSTOP_WRITE) {
ena = (req->aura.ena & req->aura_mask.ena) |
(test_bit(req->aura_id, pfvf->aura_bmap) &
~req->aura_mask.ena);
if (ena)
__set_bit(req->aura_id, pfvf->aura_bmap);
else
__clear_bit(req->aura_id, pfvf->aura_bmap);
}
}
/* Set pool bitmap if pool hw context is enabled */
if (req->ctype == NPA_AQ_CTYPE_POOL) {
if (req->op == NPA_AQ_INSTOP_INIT && req->pool.ena)
__set_bit(req->aura_id, pfvf->pool_bmap);
if (req->op == NPA_AQ_INSTOP_WRITE) {
ena = (req->pool.ena & req->pool_mask.ena) |
(test_bit(req->aura_id, pfvf->pool_bmap) &
~req->pool_mask.ena);
if (ena)
__set_bit(req->aura_id, pfvf->pool_bmap);
else
__clear_bit(req->aura_id, pfvf->pool_bmap);
}
}
spin_unlock(&aq->lock);
if (rsp) {
/* Copy read context into mailbox */
if (req->op == NPA_AQ_INSTOP_READ) {
if (req->ctype == NPA_AQ_CTYPE_AURA)
memcpy(&rsp->aura, ctx,
sizeof(struct npa_aura_s));
else
memcpy(&rsp->pool, ctx,
sizeof(struct npa_pool_s));
}
}
return 0;
}
static int npa_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
struct npa_aq_enq_req aq_req;
unsigned long *bmap;
int id, cnt = 0;
int err = 0, rc;
if (!pfvf->pool_ctx || !pfvf->aura_ctx)
return NPA_AF_ERR_AQ_ENQUEUE;
memset(&aq_req, 0, sizeof(struct npa_aq_enq_req));
aq_req.hdr.pcifunc = req->hdr.pcifunc;
if (req->ctype == NPA_AQ_CTYPE_POOL) {
aq_req.pool.ena = 0;
aq_req.pool_mask.ena = 1;
cnt = pfvf->pool_ctx->qsize;
bmap = pfvf->pool_bmap;
} else if (req->ctype == NPA_AQ_CTYPE_AURA) {
aq_req.aura.ena = 0;
aq_req.aura_mask.ena = 1;
aq_req.aura.bp_ena = 0;
aq_req.aura_mask.bp_ena = 1;
cnt = pfvf->aura_ctx->qsize;
bmap = pfvf->aura_bmap;
}
aq_req.ctype = req->ctype;
aq_req.op = NPA_AQ_INSTOP_WRITE;
for (id = 0; id < cnt; id++) {
if (!test_bit(id, bmap))
continue;
aq_req.aura_id = id;
rc = rvu_npa_aq_enq_inst(rvu, &aq_req, NULL);
if (rc) {
err = rc;
dev_err(rvu->dev, "Failed to disable %s:%d context\n",
(req->ctype == NPA_AQ_CTYPE_AURA) ?
"Aura" : "Pool", id);
}
}
return err;
}
#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
static int npa_lf_hwctx_lockdown(struct rvu *rvu, struct npa_aq_enq_req *req)
{
struct npa_aq_enq_req lock_ctx_req;
int err;
if (req->op != NPA_AQ_INSTOP_INIT)
return 0;
memset(&lock_ctx_req, 0, sizeof(struct npa_aq_enq_req));
lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc;
lock_ctx_req.ctype = req->ctype;
lock_ctx_req.op = NPA_AQ_INSTOP_LOCK;
lock_ctx_req.aura_id = req->aura_id;
err = rvu_npa_aq_enq_inst(rvu, &lock_ctx_req, NULL);
if (err)
dev_err(rvu->dev,
"PFUNC 0x%x: Failed to lock NPA context %s:%d\n",
req->hdr.pcifunc,
(req->ctype == NPA_AQ_CTYPE_AURA) ?
"Aura" : "Pool", req->aura_id);
return err;
}
int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu,
struct npa_aq_enq_req *req,
struct npa_aq_enq_rsp *rsp)
{
int err;
err = rvu_npa_aq_enq_inst(rvu, req, rsp);
if (!err)
err = npa_lf_hwctx_lockdown(rvu, req);
return err;
}
#else
int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu,
struct npa_aq_enq_req *req,
struct npa_aq_enq_rsp *rsp)
{
return rvu_npa_aq_enq_inst(rvu, req, rsp);
}
#endif
int rvu_mbox_handler_npa_hwctx_disable(struct rvu *rvu,
struct hwctx_disable_req *req,
struct msg_rsp *rsp)
{
return npa_lf_hwctx_disable(rvu, req);
}
static void npa_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
{
kfree(pfvf->aura_bmap);
pfvf->aura_bmap = NULL;
qmem_free(rvu->dev, pfvf->aura_ctx);
pfvf->aura_ctx = NULL;
kfree(pfvf->pool_bmap);
pfvf->pool_bmap = NULL;
qmem_free(rvu->dev, pfvf->pool_ctx);
pfvf->pool_ctx = NULL;
qmem_free(rvu->dev, pfvf->npa_qints_ctx);
pfvf->npa_qints_ctx = NULL;
}
int rvu_mbox_handler_npa_lf_alloc(struct rvu *rvu,
struct npa_lf_alloc_req *req,
struct npa_lf_alloc_rsp *rsp)
{
int npalf, qints, hwctx_size, err, rc = 0;
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
struct rvu_block *block;
struct rvu_pfvf *pfvf;
u64 cfg, ctx_cfg;
int blkaddr;
if (req->aura_sz > NPA_AURA_SZ_MAX ||
req->aura_sz == NPA_AURA_SZ_0 || !req->nr_pools)
return NPA_AF_ERR_PARAM;
if (req->way_mask)
req->way_mask &= 0xFFFF;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
if (!pfvf->npalf || blkaddr < 0)
return NPA_AF_ERR_AF_LF_INVALID;
block = &hw->block[blkaddr];
npalf = rvu_get_lf(rvu, block, pcifunc, 0);
if (npalf < 0)
return NPA_AF_ERR_AF_LF_INVALID;
/* Reset this NPA LF */
err = rvu_lf_reset(rvu, block, npalf);
if (err) {
dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
return NPA_AF_ERR_LF_RESET;
}
ctx_cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST1);
/* Alloc memory for aura HW contexts */
hwctx_size = 1UL << (ctx_cfg & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->aura_ctx,
NPA_AURA_COUNT(req->aura_sz), hwctx_size);
if (err)
goto free_mem;
pfvf->aura_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
GFP_KERNEL);
if (!pfvf->aura_bmap)
goto free_mem;
/* Alloc memory for pool HW contexts */
hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->pool_ctx, req->nr_pools, hwctx_size);
if (err)
goto free_mem;
pfvf->pool_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
GFP_KERNEL);
if (!pfvf->pool_bmap)
goto free_mem;
/* Get no of queue interrupts supported */
cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
qints = (cfg >> 28) & 0xFFF;
/* Alloc memory for Qints HW contexts */
hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
err = qmem_alloc(rvu->dev, &pfvf->npa_qints_ctx, qints, hwctx_size);
if (err)
goto free_mem;
cfg = rvu_read64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf));
/* Clear way partition mask and set aura offset to '0' */
cfg &= ~(BIT_ULL(34) - 1);
/* Set aura size & enable caching of contexts */
cfg |= (req->aura_sz << 16) | BIT_ULL(34) | req->way_mask;
rvu_write64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf), cfg);
/* Configure aura HW context's base */
rvu_write64(rvu, blkaddr, NPA_AF_LFX_LOC_AURAS_BASE(npalf),
(u64)pfvf->aura_ctx->iova);
/* Enable caching of qints hw context */
rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_CFG(npalf),
BIT_ULL(36) | req->way_mask << 20);
rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_BASE(npalf),
(u64)pfvf->npa_qints_ctx->iova);
goto exit;
free_mem:
npa_ctx_free(rvu, pfvf);
rc = -ENOMEM;
exit:
/* set stack page info */
cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
rsp->stack_pg_ptrs = (cfg >> 8) & 0xFF;
rsp->stack_pg_bytes = cfg & 0xFF;
rsp->qints = (cfg >> 28) & 0xFFF;
if (!is_rvu_otx2(rvu)) {
cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL);
rsp->cache_lines = (cfg >> 1) & 0x3F;
}
return rc;
}
int rvu_mbox_handler_npa_lf_free(struct rvu *rvu, struct msg_req *req,
struct msg_rsp *rsp)
{
struct rvu_hwinfo *hw = rvu->hw;
u16 pcifunc = req->hdr.pcifunc;
struct rvu_block *block;
struct rvu_pfvf *pfvf;
int npalf, err;
int blkaddr;
pfvf = rvu_get_pfvf(rvu, pcifunc);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
if (!pfvf->npalf || blkaddr < 0)
return NPA_AF_ERR_AF_LF_INVALID;
block = &hw->block[blkaddr];
npalf = rvu_get_lf(rvu, block, pcifunc, 0);
if (npalf < 0)
return NPA_AF_ERR_AF_LF_INVALID;
/* Reset this NPA LF */
err = rvu_lf_reset(rvu, block, npalf);
if (err) {
dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
return NPA_AF_ERR_LF_RESET;
}
npa_ctx_free(rvu, pfvf);
return 0;
}
static int npa_aq_init(struct rvu *rvu, struct rvu_block *block)
{
u64 cfg;
int err;
/* Set admin queue endianness */
cfg = rvu_read64(rvu, block->addr, NPA_AF_GEN_CFG);
#ifdef __BIG_ENDIAN
cfg |= BIT_ULL(1);
rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
#else
cfg &= ~BIT_ULL(1);
rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
#endif
/* Do not bypass NDC cache */
cfg = rvu_read64(rvu, block->addr, NPA_AF_NDC_CFG);
cfg &= ~0x03DULL;
#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
/* Disable caching of stack pages */
cfg |= 0x10ULL;
#endif
rvu_write64(rvu, block->addr, NPA_AF_NDC_CFG, cfg);
/* For CN10K NPA BATCH DMA set 35 cache lines */
if (!is_rvu_otx2(rvu)) {
cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL);
cfg &= ~0x7EULL;
cfg |= BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1);
rvu_write64(rvu, block->addr, NPA_AF_BATCH_CTL, cfg);
}
/* Result structure can be followed by Aura/Pool context at
* RES + 128bytes and a write mask at RES + 256 bytes, depending on
* operation type. Alloc sufficient result memory for all operations.
*/
err = rvu_aq_alloc(rvu, &block->aq,
Q_COUNT(AQ_SIZE), sizeof(struct npa_aq_inst_s),
ALIGN(sizeof(struct npa_aq_res_s), 128) + 256);
if (err)
return err;
rvu_write64(rvu, block->addr, NPA_AF_AQ_CFG, AQ_SIZE);
rvu_write64(rvu, block->addr,
NPA_AF_AQ_BASE, (u64)block->aq->inst->iova);
return 0;
}
int rvu_npa_init(struct rvu *rvu)
{
struct rvu_hwinfo *hw = rvu->hw;
int blkaddr;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
if (blkaddr < 0)
return 0;
/* Initialize admin queue */
return npa_aq_init(rvu, &hw->block[blkaddr]);
}
void rvu_npa_freemem(struct rvu *rvu)
{
struct rvu_hwinfo *hw = rvu->hw;
struct rvu_block *block;
int blkaddr;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
if (blkaddr < 0)
return;
block = &hw->block[blkaddr];
rvu_aq_free(rvu, block->aq);
}
void rvu_npa_lf_teardown(struct rvu *rvu, u16 pcifunc, int npalf)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
struct hwctx_disable_req ctx_req;
/* Disable all pools */
ctx_req.hdr.pcifunc = pcifunc;
ctx_req.ctype = NPA_AQ_CTYPE_POOL;
npa_lf_hwctx_disable(rvu, &ctx_req);
/* Disable all auras */
ctx_req.ctype = NPA_AQ_CTYPE_AURA;
npa_lf_hwctx_disable(rvu, &ctx_req);
npa_ctx_free(rvu, pfvf);
}
/* Due to an Hardware errata, in some corner cases, AQ context lock
* operations can result in a NDC way getting into an illegal state
* of not valid but locked.
*
* This API solves the problem by clearing the lock bit of the NDC block.
* The operation needs to be done for each line of all the NDC banks.
*/
int rvu_ndc_fix_locked_cacheline(struct rvu *rvu, int blkaddr)
{
int bank, max_bank, line, max_line, err;
u64 reg, ndc_af_const;
/* Set the ENABLE bit(63) to '0' */
reg = rvu_read64(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL);
rvu_write64(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL, reg & GENMASK_ULL(62, 0));
/* Poll until the BUSY bits(47:32) are set to '0' */
err = rvu_poll_reg(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL, GENMASK_ULL(47, 32), true);
if (err) {
dev_err(rvu->dev, "Timed out while polling for NDC CAM busy bits.\n");
return err;
}
ndc_af_const = rvu_read64(rvu, blkaddr, NDC_AF_CONST);
max_bank = FIELD_GET(NDC_AF_BANK_MASK, ndc_af_const);
max_line = FIELD_GET(NDC_AF_BANK_LINE_MASK, ndc_af_const);
for (bank = 0; bank < max_bank; bank++) {
for (line = 0; line < max_line; line++) {
/* Check if 'cache line valid bit(63)' is not set
* but 'cache line lock bit(60)' is set and on
* success, reset the lock bit(60).
*/
reg = rvu_read64(rvu, blkaddr,
NDC_AF_BANKX_LINEX_METADATA(bank, line));
if (!(reg & BIT_ULL(63)) && (reg & BIT_ULL(60))) {
rvu_write64(rvu, blkaddr,
NDC_AF_BANKX_LINEX_METADATA(bank, line),
reg & ~BIT_ULL(60));
}
}
}
return 0;
}