drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ptrn.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.

#include "dr_types.h"
#include "mlx5_ifc_dr_ste_v1.h"

enum dr_ptrn_modify_hdr_action_id {
	DR_PTRN_MODIFY_HDR_ACTION_ID_NOP = 0x00,
	DR_PTRN_MODIFY_HDR_ACTION_ID_COPY = 0x05,
	DR_PTRN_MODIFY_HDR_ACTION_ID_SET = 0x06,
	DR_PTRN_MODIFY_HDR_ACTION_ID_ADD = 0x07,
	DR_PTRN_MODIFY_HDR_ACTION_ID_INSERT_INLINE = 0x0a,
};

struct mlx5dr_ptrn_mgr {
	struct mlx5dr_domain *dmn;
	struct mlx5dr_icm_pool *ptrn_icm_pool;
	/* cache for modify_header ptrn */
	struct list_head ptrn_list;
	struct mutex modify_hdr_mutex; /* protect the pattern cache */
};

/* Cache structure and functions */
static bool dr_ptrn_compare_modify_hdr(size_t cur_num_of_actions,
				       __be64 cur_hw_actions[],
				       size_t num_of_actions,
				       __be64 hw_actions[])
{
	int i;

	if (cur_num_of_actions != num_of_actions)
		return false;

	for (i = 0; i < num_of_actions; i++) {
		u8 action_id =
			MLX5_GET(ste_double_action_set_v1, &hw_actions[i], action_id);

		if (action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_COPY) {
			if (hw_actions[i] != cur_hw_actions[i])
				return false;
		} else {
			if ((__force __be32)hw_actions[i] !=
			    (__force __be32)cur_hw_actions[i])
				return false;
		}
	}

	return true;
}

static struct mlx5dr_ptrn_obj *
dr_ptrn_find_cached_pattern(struct mlx5dr_ptrn_mgr *mgr,
			    size_t num_of_actions,
			    __be64 hw_actions[])
{
	struct mlx5dr_ptrn_obj *cached_pattern;
	struct mlx5dr_ptrn_obj *tmp;

	list_for_each_entry_safe(cached_pattern, tmp, &mgr->ptrn_list, list) {
		if (dr_ptrn_compare_modify_hdr(cached_pattern->num_of_actions,
					       (__be64 *)cached_pattern->data,
					       num_of_actions,
					       hw_actions)) {
			/* Put this pattern in the head of the list,
			 * as we will probably use it more.
			 */
			list_del_init(&cached_pattern->list);
			list_add(&cached_pattern->list, &mgr->ptrn_list);
			return cached_pattern;
		}
	}

	return NULL;
}

static struct mlx5dr_ptrn_obj *
dr_ptrn_alloc_pattern(struct mlx5dr_ptrn_mgr *mgr,
		      u16 num_of_actions, u8 *data)
{
	struct mlx5dr_ptrn_obj *pattern;
	struct mlx5dr_icm_chunk *chunk;
	u32 chunk_size;
	u32 index;

	chunk_size = ilog2(roundup_pow_of_two(num_of_actions));
	/* HW modify action index granularity is at least 64B */
	chunk_size = max_t(u32, chunk_size, DR_CHUNK_SIZE_8);

	chunk = mlx5dr_icm_alloc_chunk(mgr->ptrn_icm_pool, chunk_size);
	if (!chunk)
		return NULL;

	index = (mlx5dr_icm_pool_get_chunk_icm_addr(chunk) -
		 mgr->dmn->info.caps.hdr_modify_pattern_icm_addr) /
		DR_ACTION_CACHE_LINE_SIZE;

	pattern = kzalloc(sizeof(*pattern), GFP_KERNEL);
	if (!pattern)
		goto free_chunk;

	pattern->data = kzalloc(num_of_actions * DR_MODIFY_ACTION_SIZE *
				sizeof(*pattern->data), GFP_KERNEL);
	if (!pattern->data)
		goto free_pattern;

	memcpy(pattern->data, data, num_of_actions * DR_MODIFY_ACTION_SIZE);
	pattern->chunk = chunk;
	pattern->index = index;
	pattern->num_of_actions = num_of_actions;

	list_add(&pattern->list, &mgr->ptrn_list);
	refcount_set(&pattern->refcount, 1);

	return pattern;

free_pattern:
	kfree(pattern);
free_chunk:
	mlx5dr_icm_free_chunk(chunk);
	return NULL;
}

static void
dr_ptrn_free_pattern(struct mlx5dr_ptrn_obj *pattern)
{
	list_del(&pattern->list);
	mlx5dr_icm_free_chunk(pattern->chunk);
	kfree(pattern->data);
	kfree(pattern);
}

struct mlx5dr_ptrn_obj *
mlx5dr_ptrn_cache_get_pattern(struct mlx5dr_ptrn_mgr *mgr,
			      u16 num_of_actions,
			      u8 *data)
{
	struct mlx5dr_ptrn_obj *pattern;
	u64 *hw_actions;
	u8 action_id;
	int i;

	mutex_lock(&mgr->modify_hdr_mutex);
	pattern = dr_ptrn_find_cached_pattern(mgr,
					      num_of_actions,
					      (__be64 *)data);
	if (!pattern) {
		/* Alloc and add new pattern to cache */
		pattern = dr_ptrn_alloc_pattern(mgr, num_of_actions, data);
		if (!pattern)
			goto out_unlock;

		hw_actions = (u64 *)pattern->data;
		/* Here we mask the pattern data to create a valid pattern
		 * since we do an OR operation between the arg and pattern
		 */
		for (i = 0; i < num_of_actions; i++) {
			action_id = MLX5_GET(ste_double_action_set_v1, &hw_actions[i], action_id);

			if (action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_SET ||
			    action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_ADD ||
			    action_id == DR_PTRN_MODIFY_HDR_ACTION_ID_INSERT_INLINE)
				MLX5_SET(ste_double_action_set_v1, &hw_actions[i], inline_data, 0);
		}

		if (mlx5dr_send_postsend_pattern(mgr->dmn, pattern->chunk,
						 num_of_actions, pattern->data)) {
			refcount_dec(&pattern->refcount);
			goto free_pattern;
		}
	} else {
		refcount_inc(&pattern->refcount);
	}

	mutex_unlock(&mgr->modify_hdr_mutex);

	return pattern;

free_pattern:
	dr_ptrn_free_pattern(pattern);
out_unlock:
	mutex_unlock(&mgr->modify_hdr_mutex);
	return NULL;
}

void
mlx5dr_ptrn_cache_put_pattern(struct mlx5dr_ptrn_mgr *mgr,
			      struct mlx5dr_ptrn_obj *pattern)
{
	mutex_lock(&mgr->modify_hdr_mutex);

	if (refcount_dec_and_test(&pattern->refcount))
		dr_ptrn_free_pattern(pattern);

	mutex_unlock(&mgr->modify_hdr_mutex);
}

struct mlx5dr_ptrn_mgr *mlx5dr_ptrn_mgr_create(struct mlx5dr_domain *dmn)
{
	struct mlx5dr_ptrn_mgr *mgr;

	if (!mlx5dr_domain_is_support_ptrn_arg(dmn))
		return NULL;

	mgr = kzalloc(sizeof(*mgr), GFP_KERNEL);
	if (!mgr)
		return NULL;

	mgr->dmn = dmn;
	mgr->ptrn_icm_pool = mlx5dr_icm_pool_create(dmn, DR_ICM_TYPE_MODIFY_HDR_PTRN);
	if (!mgr->ptrn_icm_pool) {
		mlx5dr_err(dmn, "Couldn't get modify-header-pattern memory\n");
		goto free_mgr;
	}

	INIT_LIST_HEAD(&mgr->ptrn_list);
	mutex_init(&mgr->modify_hdr_mutex);

	return mgr;

free_mgr:
	kfree(mgr);
	return NULL;
}

void mlx5dr_ptrn_mgr_destroy(struct mlx5dr_ptrn_mgr *mgr)
{
	struct mlx5dr_ptrn_obj *pattern;
	struct mlx5dr_ptrn_obj *tmp;

	if (!mgr)
		return;

	WARN_ON(!list_empty(&mgr->ptrn_list));

	list_for_each_entry_safe(pattern, tmp, &mgr->ptrn_list, list) {
		list_del(&pattern->list);
		kfree(pattern->data);
		kfree(pattern);
	}

	mlx5dr_icm_pool_destroy(mgr->ptrn_icm_pool);
	mutex_destroy(&mgr->modify_hdr_mutex);
	kfree(mgr);
}