btf_encoder.c

/*
  SPDX-License-Identifier: GPL-2.0-only

  Copyright (C) 2019 Facebook

  Derived from ctf_encoder.c, which is:

  Copyright (C) Arnaldo Carvalho de Melo <acme@redhat.com>
  Copyright (C) Red Hat Inc
 */

#include <linux/btf.h>
#include "dwarves.h"
#include "elf_symtab.h"
#include "btf_encoder.h"
#include "gobuffer.h"

#include <bpf/btf.h>
#include <bpf/libbpf.h>
#include <ctype.h> /* for isalpha() and isalnum() */
#include <stdlib.h> /* for qsort() and bsearch() */
#include <inttypes.h>
#include <limits.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <unistd.h>

#include <errno.h>
#include <stdint.h>
#include <search.h> /* for tsearch(), tfind() and tdestroy() */
#include <pthread.h>

#define BTF_ENCODER_MAX_PROTO	512

/* state used to do later encoding of saved functions */
struct btf_encoder_state {
	uint32_t type_id_off;
	bool got_proto;
	char proto[BTF_ENCODER_MAX_PROTO];
};

struct elf_function {
	const char	*name;
	bool		 generated;
	size_t		prefixlen;
	struct function	*function;
	struct btf_encoder_state state;
};

#define MAX_PERCPU_VAR_CNT 4096

struct var_info {
	uint64_t    addr;
	const char *name;
	uint32_t    sz;
};

/*
 * cu: cu being processed.
 */
struct btf_encoder {
	struct list_head  node;
	struct btf        *btf;
	struct cu         *cu;
	struct gobuffer   percpu_secinfo;
	const char	  *filename;
	struct elf_symtab *symtab;
	uint32_t	  type_id_off;
	int		  saved_func_cnt;
	bool		  has_index_type,
			  need_index_type,
			  skip_encoding_vars,
			  raw_output,
			  verbose,
			  force,
			  gen_floats,
			  is_rel;
	uint32_t	  array_index_id;
	struct {
		struct var_info vars[MAX_PERCPU_VAR_CNT];
		int		var_cnt;
		uint32_t	shndx;
		uint64_t	base_addr;
		uint64_t	sec_sz;
	} percpu;
	struct {
		struct elf_function *entries;
		int		    allocated;
		int		    cnt;
		int		    suffix_cnt; /* number of .isra, .part etc */
	} functions;
};

static LIST_HEAD(encoders);
static pthread_mutex_t encoders__lock = PTHREAD_MUTEX_INITIALIZER;

static void btf_encoder__add_saved_funcs(struct btf_encoder *encoder);

/* mutex only needed for add/delete, as this can happen in multiple encoding
 * threads.  Traversal of the list is currently confined to thread collection.
 */

#define btf_encoders__for_each_encoder(encoder)		\
	list_for_each_entry(encoder, &encoders, node)

static void btf_encoders__add(struct btf_encoder *encoder)
{
	pthread_mutex_lock(&encoders__lock);
	list_add_tail(&encoder->node, &encoders);
	pthread_mutex_unlock(&encoders__lock);
}

static void btf_encoders__delete(struct btf_encoder *encoder)
{
	struct btf_encoder *existing = NULL;

	pthread_mutex_lock(&encoders__lock);
	/* encoder may not have been added to list yet; check. */
	btf_encoders__for_each_encoder(existing) {
		if (encoder == existing)
			break;
	}
	if (encoder == existing)
		list_del(&encoder->node);
	pthread_mutex_unlock(&encoders__lock);
}

#define PERCPU_SECTION ".data..percpu"

/*
 * This depends on the GNU extension to eliminate the stray comma in the zero
 * arguments case.
 *
 * The difference between elf_errmsg(-1) and elf_errmsg(elf_errno()) is that the
 * latter clears the current error.
 */
#define elf_error(fmt, ...) \
        fprintf(stderr, "%s: " fmt ": %s.\n", __func__, ##__VA_ARGS__, elf_errmsg(-1))

/*
 * This depends on the GNU extension to eliminate the stray comma in the zero
 * arguments case.
 *
 * The difference between elf_errmsg(-1) and elf_errmsg(elf_errno()) is that the
 * latter clears the current error.
 */
#define elf_error(fmt, ...) \
	fprintf(stderr, "%s: " fmt ": %s.\n", __func__, ##__VA_ARGS__, elf_errmsg(-1))

static int btf_var_secinfo_cmp(const void *a, const void *b)
{
	const struct btf_var_secinfo *av = a;
	const struct btf_var_secinfo *bv = b;

	return av->offset - bv->offset;
}

#define BITS_PER_BYTE 8
#define BITS_PER_BYTE_MASK (BITS_PER_BYTE - 1)
#define BITS_PER_BYTE_MASKED(bits) ((bits) & BITS_PER_BYTE_MASK)
#define BITS_ROUNDDOWN_BYTES(bits) ((bits) >> 3)
#define BITS_ROUNDUP_BYTES(bits) (BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))

static const char * const btf_kind_str[] = {
	[BTF_KIND_UNKN]		= "UNKNOWN",
	[BTF_KIND_INT]		= "INT",
	[BTF_KIND_PTR]		= "PTR",
	[BTF_KIND_ARRAY]	= "ARRAY",
	[BTF_KIND_STRUCT]	= "STRUCT",
	[BTF_KIND_UNION]	= "UNION",
	[BTF_KIND_ENUM]		= "ENUM",
	[BTF_KIND_FWD]		= "FWD",
	[BTF_KIND_TYPEDEF]	= "TYPEDEF",
	[BTF_KIND_VOLATILE]	= "VOLATILE",
	[BTF_KIND_CONST]	= "CONST",
	[BTF_KIND_RESTRICT]	= "RESTRICT",
	[BTF_KIND_FUNC]		= "FUNC",
	[BTF_KIND_FUNC_PROTO]	= "FUNC_PROTO",
	[BTF_KIND_VAR]          = "VAR",
	[BTF_KIND_DATASEC]      = "DATASEC",
	[BTF_KIND_FLOAT]        = "FLOAT",
	[BTF_KIND_DECL_TAG]     = "DECL_TAG",
	[BTF_KIND_TYPE_TAG]     = "TYPE_TAG",
	[BTF_KIND_ENUM64]	= "ENUM64",
};

static const char *btf__printable_name(const struct btf *btf, uint32_t offset)
{
	if (!offset)
		return "(anon)";
	else
		return btf__str_by_offset(btf, offset);
}

static const char * btf__int_encoding_str(uint8_t encoding)
{
	if (encoding == 0)
		return "(none)";
	else if (encoding == BTF_INT_SIGNED)
		return "SIGNED";
	else if (encoding == BTF_INT_CHAR)
		return "CHAR";
	else if (encoding == BTF_INT_BOOL)
		return "BOOL";
	else
		return "UNKN";
}

__attribute ((format (printf, 5, 6)))
static void btf__log_err(const struct btf *btf, int kind, const char *name,
			 bool output_cr, const char *fmt, ...)
{
	fprintf(stderr, "[%u] %s %s", btf__type_cnt(btf),
		btf_kind_str[kind], name ?: "(anon)");

	if (fmt && *fmt) {
		va_list ap;

		fprintf(stderr, " ");
		va_start(ap, fmt);
		vfprintf(stderr, fmt, ap);
		va_end(ap);
	}

	if (output_cr)
		fprintf(stderr, "\n");
}

__attribute ((format (printf, 5, 6)))
static void btf_encoder__log_type(const struct btf_encoder *encoder, const struct btf_type *t,
				  bool err, bool output_cr, const char *fmt, ...)
{
	const struct btf *btf = encoder->btf;
	uint8_t kind;
	FILE *out;

	if (!encoder->verbose && !err)
		return;

	kind = BTF_INFO_KIND(t->info);
	out = err ? stderr : stdout;

	fprintf(out, "[%u] %s %s",
		btf__type_cnt(btf) - 1, btf_kind_str[kind],
		btf__printable_name(btf, t->name_off));

	if (fmt && *fmt) {
		va_list ap;

		fprintf(out, " ");
		va_start(ap, fmt);
		vfprintf(out, fmt, ap);
		va_end(ap);
	}

	if (output_cr)
		fprintf(out, "\n");
}

__attribute ((format (printf, 5, 6)))
static void btf_encoder__log_member(const struct btf_encoder *encoder, const struct btf_type *t,
				    const struct btf_member *member, bool err, const char *fmt, ...)
{
	const struct btf *btf = encoder->btf;
	FILE *out;

	if (!encoder->verbose && !err)
		return;

	out = err ? stderr : stdout;

	if (btf_kflag(t))
		fprintf(out, "\t%s type_id=%u bitfield_size=%u bits_offset=%u",
			btf__printable_name(btf, member->name_off),
			member->type,
			BTF_MEMBER_BITFIELD_SIZE(member->offset),
			BTF_MEMBER_BIT_OFFSET(member->offset));
	else
		fprintf(out, "\t%s type_id=%u bits_offset=%u",
			btf__printable_name(btf, member->name_off),
			member->type,
			member->offset);

	if (fmt && *fmt) {
		va_list ap;

		fprintf(out, " ");
		va_start(ap, fmt);
		vfprintf(out, fmt, ap);
		va_end(ap);
	}

	fprintf(out, "\n");
}

__attribute ((format (printf, 6, 7)))
static void btf_encoder__log_func_param(struct btf_encoder *encoder, const char *name, uint32_t type,
					bool err, bool is_last_param, const char *fmt, ...)
{
	FILE *out;

	if (!encoder->verbose && !err)
		return;

	out = err ? stderr : stdout;

	if (is_last_param && !type)
		fprintf(out, "vararg)\n");
	else
		fprintf(out, "%u %s%s", type, name, is_last_param ? ")\n" : ", ");

	if (fmt && *fmt) {
		va_list ap;

		fprintf(out, " ");
		va_start(ap, fmt);
		vfprintf(out, fmt, ap);
		va_end(ap);
	}
}

static int32_t btf_encoder__add_float(struct btf_encoder *encoder, const struct base_type *bt, const char *name)
{
	int32_t id = btf__add_float(encoder->btf, name, BITS_ROUNDUP_BYTES(bt->bit_size));

	if (id < 0) {
		btf__log_err(encoder->btf, BTF_KIND_FLOAT, name, true, "Error emitting BTF type");
	} else {
		const struct btf_type *t;

		t = btf__type_by_id(encoder->btf, id);
		btf_encoder__log_type(encoder, t, false, true, "size=%u nr_bits=%u", t->size, bt->bit_size);
	}

	return id;
}

static int32_t btf_encoder__add_base_type(struct btf_encoder *encoder, const struct base_type *bt, const char *name)
{
	const struct btf_type *t;
	uint8_t encoding = 0;
	uint16_t byte_sz;
	int32_t id;

	if (bt->is_signed) {
		encoding = BTF_INT_SIGNED;
	} else if (bt->is_bool) {
		encoding = BTF_INT_BOOL;
	} else if (bt->float_type && encoder->gen_floats) {
		/*
		 * Encode floats as BTF_KIND_FLOAT if allowed, otherwise (in
		 * compatibility mode) encode them as BTF_KIND_INT - that's not
		 * fully correct, but that's what it used to be.
		 */
		if (bt->float_type == BT_FP_SINGLE ||
		    bt->float_type == BT_FP_DOUBLE ||
		    bt->float_type == BT_FP_LDBL)
			return btf_encoder__add_float(encoder, bt, name);
		fprintf(stderr, "Complex, interval and imaginary float types are not supported\n");
		return -1;
	}

	/* dwarf5 may emit DW_ATE_[un]signed_{num} base types where
	 * {num} is not power of 2 and may exceed 128. Such attributes
	 * are mostly used to record operation for an actual parameter
	 * or variable.
	 * For example,
	 *     DW_AT_location        (indexed (0x3c) loclist = 0x00008fb0:
	 *         [0xffffffff82808812, 0xffffffff82808817):
	 *             DW_OP_breg0 RAX+0,
	 *             DW_OP_convert (0x000e97d5) "DW_ATE_unsigned_64",
	 *             DW_OP_convert (0x000e97df) "DW_ATE_unsigned_8",
	 *             DW_OP_stack_value,
	 *             DW_OP_piece 0x1,
	 *             DW_OP_breg0 RAX+0,
	 *             DW_OP_convert (0x000e97d5) "DW_ATE_unsigned_64",
	 *             DW_OP_convert (0x000e97da) "DW_ATE_unsigned_32",
	 *             DW_OP_lit8,
	 *             DW_OP_shr,
	 *             DW_OP_convert (0x000e97da) "DW_ATE_unsigned_32",
	 *             DW_OP_convert (0x000e97e4) "DW_ATE_unsigned_24",
	 *             DW_OP_stack_value, DW_OP_piece 0x3
	 *     DW_AT_name    ("ebx")
	 *     DW_AT_decl_file       ("/linux/arch/x86/events/intel/core.c")
	 *
	 * In the above example, at some point, one unsigned_32 value
	 * is right shifted by 8 and the result is converted to unsigned_32
	 * and then unsigned_24.
	 *
	 * BTF does not need such DW_OP_* information so let us sanitize
	 * these non-regular int types to avoid libbpf/kernel complaints.
	 */
	byte_sz = BITS_ROUNDUP_BYTES(bt->bit_size);
	if (!byte_sz || (byte_sz & (byte_sz - 1))) {
		name = "__SANITIZED_FAKE_INT__";
		byte_sz = 4;
	}

	id = btf__add_int(encoder->btf, name, byte_sz, encoding);
	if (id < 0) {
		btf__log_err(encoder->btf, BTF_KIND_INT, name, true, "Error emitting BTF type");
	} else {
		t = btf__type_by_id(encoder->btf, id);
		btf_encoder__log_type(encoder, t, false, true, "size=%u nr_bits=%u encoding=%s%s",
				      t->size, bt->bit_size, btf__int_encoding_str(encoding),
				      id < 0 ? " Error in emitting BTF" : "" );
	}

	return id;
}

static int32_t btf_encoder__add_ref_type(struct btf_encoder *encoder, uint16_t kind, uint32_t type,
					 const char *name, bool kind_flag)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	int32_t id;

	switch (kind) {
	case BTF_KIND_PTR:
		id = btf__add_ptr(btf, type);
		break;
	case BTF_KIND_VOLATILE:
		id = btf__add_volatile(btf, type);
		break;
	case BTF_KIND_CONST:
		id = btf__add_const(btf, type);
		break;
	case BTF_KIND_RESTRICT:
		id = btf__add_restrict(btf, type);
		break;
	case BTF_KIND_TYPEDEF:
		id = btf__add_typedef(btf, name, type);
		break;
	case BTF_KIND_TYPE_TAG:
		id = btf__add_type_tag(btf, name, type);
		break;
	case BTF_KIND_FWD:
		id = btf__add_fwd(btf, name, kind_flag);
		break;
	case BTF_KIND_FUNC:
		id = btf__add_func(btf, name, BTF_FUNC_STATIC, type);
		break;
	default:
		btf__log_err(btf, kind, name, true, "Unexpected kind for reference");
		return -1;
	}

	if (id > 0) {
		t = btf__type_by_id(btf, id);
		if (kind == BTF_KIND_FWD)
			btf_encoder__log_type(encoder, t, false, true, "%s", kind_flag ? "union" : "struct");
		else
			btf_encoder__log_type(encoder, t, false, true, "type_id=%u", t->type);
	} else {
		btf__log_err(btf, kind, name, true, "Error emitting BTF type");
	}
	return id;
}

static int32_t btf_encoder__add_array(struct btf_encoder *encoder, uint32_t type, uint32_t index_type, uint32_t nelems)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	const struct btf_array *array;
	int32_t id;

	id = btf__add_array(btf, index_type, type, nelems);
	if (id > 0) {
		t = btf__type_by_id(btf, id);
		array = btf_array(t);
		btf_encoder__log_type(encoder, t, false, true, "type_id=%u index_type_id=%u nr_elems=%u",
				      array->type, array->index_type, array->nelems);
	} else {
		btf__log_err(btf, BTF_KIND_ARRAY, NULL, true,
			      "type_id=%u index_type_id=%u nr_elems=%u Error emitting BTF type",
			      type, index_type, nelems);
	}
	return id;
}

static int btf_encoder__add_field(struct btf_encoder *encoder, const char *name, uint32_t type, uint32_t bitfield_size, uint32_t offset)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	const struct btf_member *m;
	int err;

	err = btf__add_field(btf, name, type, offset, bitfield_size);
	t = btf__type_by_id(btf, btf__type_cnt(btf) - 1);
	if (err) {
		fprintf(stderr, "[%u] %s %s's field '%s' offset=%u bit_size=%u type=%u Error emitting field\n",
			btf__type_cnt(btf) - 1, btf_kind_str[btf_kind(t)],
			btf__printable_name(btf, t->name_off),
			name, offset, bitfield_size, type);
	} else {
		m = &btf_members(t)[btf_vlen(t) - 1];
		btf_encoder__log_member(encoder, t, m, false, NULL);
	}
	return err;
}

static int32_t btf_encoder__add_struct(struct btf_encoder *encoder, uint8_t kind, const char *name, uint32_t size)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	int32_t id;

	switch (kind) {
	case BTF_KIND_STRUCT:
		id = btf__add_struct(btf, name, size);
		break;
	case BTF_KIND_UNION:
		id = btf__add_union(btf, name, size);
		break;
	default:
		btf__log_err(btf, kind, name, true, "Unexpected kind of struct");
		return -1;
	}

	if (id < 0) {
		btf__log_err(btf, kind, name, true, "Error emitting BTF type");
	} else {
		t = btf__type_by_id(btf, id);
		btf_encoder__log_type(encoder, t, false, true, "size=%u", t->size);
	}

	return id;
}

#if LIBBPF_MAJOR_VERSION < 1
static inline int libbpf_err(int ret)
{
        if (ret < 0)
                errno = -ret;
        return ret;
}

static
int btf__add_enum64(struct btf *btf __maybe_unused, const char *name __maybe_unused,
		    __u32 byte_sz __maybe_unused, bool is_signed __maybe_unused)
{
	return  libbpf_err(-ENOTSUP);
}

static
int btf__add_enum64_value(struct btf *btf __maybe_unused, const char *name __maybe_unused,
			  __u64 value __maybe_unused)
{
	return  libbpf_err(-ENOTSUP);
}
#endif

static int32_t btf_encoder__add_enum(struct btf_encoder *encoder, const char *name, struct type *etype,
				     struct conf_load *conf_load)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	int32_t id, size;
	bool is_enum32;

	size = BITS_ROUNDUP_BYTES(etype->size);
	is_enum32 = size <= 4 || conf_load->skip_encoding_btf_enum64;
	if (is_enum32)
		id = btf__add_enum(btf, name, size);
	else
		id = btf__add_enum64(btf, name, size, etype->is_signed_enum);
	if (id > 0) {
		t = btf__type_by_id(btf, id);
		btf_encoder__log_type(encoder, t, false, true, "size=%u", t->size);
	} else {
		btf__log_err(btf, is_enum32 ? BTF_KIND_ENUM : BTF_KIND_ENUM64, name, true,
			      "size=%u Error emitting BTF type", size);
	}
	return id;
}

static int btf_encoder__add_enum_val(struct btf_encoder *encoder, const char *name, int64_t value,
				     struct type *etype, struct conf_load *conf_load)
{
	const char *fmt_str;
	int err;

	/* If enum64 is not allowed, generate enum32 with unsigned int value. In enum64-supported
	 * libbpf library, btf__add_enum_value() will set the kflag (sign bit) in common_type
	 * if the value is negative.
	 */
	if (conf_load->skip_encoding_btf_enum64)
		err = btf__add_enum_value(encoder->btf, name, (uint32_t)value);
	else if (etype->size > 32)
		err = btf__add_enum64_value(encoder->btf, name, value);
	else
		err = btf__add_enum_value(encoder->btf, name, value);

	if (!err) {
		if (encoder->verbose) {
			if (conf_load->skip_encoding_btf_enum64) {
				printf("\t%s val=%u\n", name, (uint32_t)value);
			} else {
				fmt_str = etype->is_signed_enum ? "\t%s val=%lld\n" : "\t%s val=%llu\n";
				printf(fmt_str, name, (unsigned long long)value);
			}
		}
	} else {
		if (conf_load->skip_encoding_btf_enum64) {
			fprintf(stderr, "\t%s val=%u Error emitting BTF enum value\n", name, (uint32_t)value);
		} else {
			fmt_str = etype->is_signed_enum ? "\t%s val=%lld Error emitting BTF enum value\n"
							: "\t%s val=%llu Error emitting BTF enum value\n";
			fprintf(stderr, fmt_str, name, (unsigned long long)value);
		}
	}
	return err;
}

static int32_t btf_encoder__add_func_param(struct btf_encoder *encoder, const char *name, uint32_t type, bool is_last_param)
{
	int err = btf__add_func_param(encoder->btf, name, type);

	if (!err) {
		btf_encoder__log_func_param(encoder, name, type, false, is_last_param, NULL);
		return 0;
	} else {
		btf_encoder__log_func_param(encoder, name, type, true, is_last_param, "Error adding func param");
		return -1;
	}
}

static int32_t btf_encoder__tag_type(struct btf_encoder *encoder, uint32_t tag_type)
{
	if (tag_type == 0)
		return 0;

	return encoder->type_id_off + tag_type;
}

static int32_t btf_encoder__add_func_proto(struct btf_encoder *encoder, struct ftype *ftype)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	struct parameter *param;
	uint16_t nr_params, param_idx;
	int32_t id, type_id;

	/* add btf_type for func_proto */
	nr_params = ftype->nr_parms + (ftype->unspec_parms ? 1 : 0);
	type_id = btf_encoder__tag_type(encoder, ftype->tag.type);

	id = btf__add_func_proto(btf, type_id);
	if (id > 0) {
		t = btf__type_by_id(btf, id);
		btf_encoder__log_type(encoder, t, false, false, "return=%u args=(%s", t->type, !nr_params ? "void)\n" : "");
	} else {
		btf__log_err(btf, BTF_KIND_FUNC_PROTO, NULL, true,
			      "return=%u vlen=%u Error emitting BTF type",
			      type_id, nr_params);
		return id;
	}

	/* add parameters */
	param_idx = 0;
	ftype__for_each_parameter(ftype, param) {
		const char *name = parameter__name(param);

		type_id = param->tag.type == 0 ? 0 : encoder->type_id_off + param->tag.type;
		++param_idx;
		if (btf_encoder__add_func_param(encoder, name, type_id, param_idx == nr_params))
			return -1;
	}

	++param_idx;
	if (ftype->unspec_parms)
		if (btf_encoder__add_func_param(encoder, NULL, 0, param_idx == nr_params))
			return -1;

	return id;
}

static int32_t btf_encoder__add_var(struct btf_encoder *encoder, uint32_t type, const char *name, uint32_t linkage)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	int32_t id;

	id = btf__add_var(btf, name, linkage, type);
	if (id > 0) {
		t = btf__type_by_id(btf, id);
		btf_encoder__log_type(encoder, t, false, true, "type=%u linkage=%u", t->type, btf_var(t)->linkage);
	} else {
		btf__log_err(btf, BTF_KIND_VAR, name, true,
			      "type=%u linkage=%u Error emitting BTF type",
			      type, linkage);
	}
	return id;
}

static int32_t btf_encoder__add_var_secinfo(struct btf_encoder *encoder, uint32_t type,
				     uint32_t offset, uint32_t size)
{
	struct btf_var_secinfo si = {
		.type = type,
		.offset = offset,
		.size = size,
	};
	return gobuffer__add(&encoder->percpu_secinfo, &si, sizeof(si));
}

int32_t btf_encoder__add_encoder(struct btf_encoder *encoder, struct btf_encoder *other)
{
	struct gobuffer *var_secinfo_buf = &other->percpu_secinfo;
	size_t sz = gobuffer__size(var_secinfo_buf);
	uint16_t nr_var_secinfo = sz / sizeof(struct btf_var_secinfo);
	uint32_t type_id;
	uint32_t next_type_id = btf__type_cnt(encoder->btf);
	int32_t i, id;
	struct btf_var_secinfo *vsi;

	if (encoder == other)
		return 0;

	btf_encoder__add_saved_funcs(other);

	for (i = 0; i < nr_var_secinfo; i++) {
		vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + i;
		type_id = next_type_id + vsi->type - 1; /* Type ID starts from 1 */
		id = btf_encoder__add_var_secinfo(encoder, type_id, vsi->offset, vsi->size);
		if (id < 0)
			return id;
	}

	return btf__add_btf(encoder->btf, other->btf);
}

static int32_t btf_encoder__add_datasec(struct btf_encoder *encoder, const char *section_name)
{
	struct gobuffer *var_secinfo_buf = &encoder->percpu_secinfo;
	struct btf *btf = encoder->btf;
	size_t sz = gobuffer__size(var_secinfo_buf);
	uint16_t nr_var_secinfo = sz / sizeof(struct btf_var_secinfo);
	struct btf_var_secinfo *last_vsi, *vsi;
	const struct btf_type *t;
	uint32_t datasec_sz;
	int32_t err, id, i;

	qsort(var_secinfo_buf->entries, nr_var_secinfo,
	      sizeof(struct btf_var_secinfo), btf_var_secinfo_cmp);

	last_vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + nr_var_secinfo - 1;
	datasec_sz = last_vsi->offset + last_vsi->size;

	id = btf__add_datasec(btf, section_name, datasec_sz);
	if (id < 0) {
		btf__log_err(btf, BTF_KIND_DATASEC, section_name, true,
				 "size=%u vlen=%u Error emitting BTF type",
				 datasec_sz, nr_var_secinfo);
	} else {
		t = btf__type_by_id(btf, id);
		btf_encoder__log_type(encoder, t, false, true, "size=%u vlen=%u", t->size, nr_var_secinfo);
	}

	for (i = 0; i < nr_var_secinfo; i++) {
		vsi = (struct btf_var_secinfo *)var_secinfo_buf->entries + i;
		err = btf__add_datasec_var_info(btf, vsi->type, vsi->offset, vsi->size);
		if (!err) {
			if (encoder->verbose)
				printf("\ttype=%u offset=%u size=%u\n",
				       vsi->type, vsi->offset, vsi->size);
		} else {
			fprintf(stderr, "\ttype=%u offset=%u size=%u Error emitting BTF datasec var info\n",
				       vsi->type, vsi->offset, vsi->size);
			return -1;
		}
	}

	return id;
}

static int32_t btf_encoder__add_decl_tag(struct btf_encoder *encoder, const char *value, uint32_t type,
					 int component_idx)
{
	struct btf *btf = encoder->btf;
	const struct btf_type *t;
	int32_t id;

	id = btf__add_decl_tag(btf, value, type, component_idx);
	if (id > 0) {
		t = btf__type_by_id(btf, id);
		btf_encoder__log_type(encoder, t, false, true, "type_id=%u component_idx=%d",
				      t->type, component_idx);
	} else {
		btf__log_err(btf, BTF_KIND_DECL_TAG, value, true, "component_idx=%d Error emitting BTF type",
			     component_idx);
	}

	return id;
}

static bool proto__get(struct function *func, char *proto, size_t len)
{
	const struct conf_fprintf conf = {
						.name_spacing = 23,
						.type_spacing = 26,
						.emit_stats = 0,
						.no_parm_names = 1,
						.skip_emitting_errors = 1,
						.skip_emitting_modifier = 1,
					};

	return function__prototype_conf(func, func->priv, &conf, proto, len) != NULL;
}

static bool funcs__match(struct btf_encoder *encoder, struct elf_function *func, struct function *f2)
{
	char proto[BTF_ENCODER_MAX_PROTO];
	struct function *f1 = func->function;
	const char *name;

	if (!f1)
		return false;

	name = function__name(f1);

	if (f1->proto.nr_parms != f2->proto.nr_parms) {
		if (encoder->verbose)
			printf("function mismatch for '%s'(%s): %d params != %d params\n",
			       name, f1->alias ?: name,
			       f1->proto.nr_parms, f2->proto.nr_parms);
		return false;
	}
	if (f1->proto.nr_parms == 0)
		return true;

	if (f1->proto.tag.type == f2->proto.tag.type)
		return true;

	if (!func->state.got_proto)
		func->state.got_proto = proto__get(f1, func->state.proto, sizeof(func->state.proto));

	if (proto__get(f2, proto, sizeof(proto))) {
		if (strcmp(func->state.proto, proto) != 0) {
			if (encoder->verbose)
				printf("function mismatch for '%s'('%s'): '%s' != '%s'\n",
				       name, f1->alias ?: name,
				       func->state.proto, proto);
			return false;
		}
	}
	return true;
}

static int32_t btf_encoder__save_func(struct btf_encoder *encoder, struct function *fn, struct elf_function *func)
{
	fn->priv = encoder->cu;
	if (func->function) {
		struct function *existing = func->function;

		/* If saving and we find an existing entry, we want to merge
		 * observations across both functions, checking that the
		 * "seen optimized parameters", "inconsistent prototype"
		 * and "unexpected register" status is reflected in the
		 * the func entry.
		 * If the entry is new, record encoder state required
		 * to add the local function later (encoder + type_id_off)
		 * such that we can add the function later.
		 */
		existing->proto.optimized_parms |= fn->proto.optimized_parms;
		existing->proto.unexpected_reg |= fn->proto.unexpected_reg;
		if (!existing->proto.unexpected_reg && !existing->proto.inconsistent_proto &&
		     !funcs__match(encoder, func, fn))
			existing->proto.inconsistent_proto = 1;
	} else {
		func->state.type_id_off = encoder->type_id_off;
		func->function = fn;
		encoder->saved_func_cnt++;
	}
	return 0;
}

static int32_t btf_encoder__add_func(struct btf_encoder *encoder, struct function *fn)
{
	int btf_fnproto_id, btf_fn_id, tag_type_id;
	struct llvm_annotation *annot;
	const char *name;

	btf_fnproto_id = btf_encoder__add_func_proto(encoder, &fn->proto);
	name = function__name(fn);
	btf_fn_id = btf_encoder__add_ref_type(encoder, BTF_KIND_FUNC, btf_fnproto_id, name, false);
	if (btf_fnproto_id < 0 || btf_fn_id < 0) {
		printf("error: failed to encode function '%s'\n", function__name(fn));
		return -1;
	}
	list_for_each_entry(annot, &fn->annots, node) {
		tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, btf_fn_id,
							annot->component_idx);
		if (tag_type_id < 0) {
			fprintf(stderr, "error: failed to encode tag '%s' to func %s with component_idx %d\n",
				annot->value, name, annot->component_idx);
			return -1;
		}
	}
	return 0;
}

static void btf_encoder__add_saved_funcs(struct btf_encoder *encoder)
{
	int i;

	for (i = 0; i < encoder->functions.cnt; i++) {
		struct elf_function *func = &encoder->functions.entries[i];
		struct function *fn = func->function;
		struct btf_encoder *other_encoder;

		if (!fn || fn->proto.processed)
			continue;

		/* merge optimized-out status across encoders; since each
		 * encoder has the same elf symbol table we can use the
		 * same index to access the same elf symbol.
		 */
		btf_encoders__for_each_encoder(other_encoder) {
			struct function *other_fn;

			if (other_encoder == encoder)
				continue;

			other_fn = other_encoder->functions.entries[i].function;
			if (!other_fn)
				continue;
			fn->proto.optimized_parms |= other_fn->proto.optimized_parms;
			fn->proto.unexpected_reg |= other_fn->proto.unexpected_reg;
			if (other_fn->proto.inconsistent_proto)
				fn->proto.inconsistent_proto = 1;
			if (!fn->proto.unexpected_reg && !fn->proto.inconsistent_proto &&
			    !funcs__match(encoder, func, other_fn))
				fn->proto.inconsistent_proto = 1;
			other_fn->proto.processed = 1;
		}
		/* do not exclude functions with optimized-out parameters; they
		 * may still be _called_ with the right parameter values, they
		 * just do not _use_ them.  Only exclude functions with
		 * unexpected register use or multiple inconsistent prototypes.
		 */
		if (fn->proto.unexpected_reg || fn->proto.inconsistent_proto) {
			if (encoder->verbose) {
				const char *name = function__name(fn);

				printf("skipping addition of '%s'(%s) due to %s\n",
				       name, fn->alias ?: name,
				       fn->proto.unexpected_reg ? "unexpected register used for parameter" :
								   "multiple inconsistent function prototypes");
			}
		} else {
			encoder->type_id_off = func->state.type_id_off;
			btf_encoder__add_func(encoder, fn);
		}
		fn->proto.processed = 1;
	}
}

/*
 * This corresponds to the same macro defined in
 * include/linux/kallsyms.h
 */
#define KSYM_NAME_LEN 128

static int functions_cmp(const void *_a, const void *_b)
{
	const struct elf_function *a = _a;
	const struct elf_function *b = _b;

	/* if search key allows prefix match, verify target has matching
	 * prefix len and prefix matches.
	 */
	if (a->prefixlen && a->prefixlen == b->prefixlen)
		return strncmp(a->name, b->name, b->prefixlen);
	return strcmp(a->name, b->name);
}

#ifndef max
#define max(x, y) ((x) < (y) ? (y) : (x))
#endif

static int btf_encoder__collect_function(struct btf_encoder *encoder, GElf_Sym *sym)
{
	struct elf_function *new;
	const char *name;

	if (elf_sym__type(sym) != STT_FUNC)
		return 0;
	name = elf_sym__name(sym, encoder->symtab);
	if (!name)
		return 0;

	if (encoder->functions.cnt == encoder->functions.allocated) {
		encoder->functions.allocated = max(1000, encoder->functions.allocated * 3 / 2);
		new = realloc(encoder->functions.entries, encoder->functions.allocated * sizeof(*encoder->functions.entries));
		if (!new) {
			/*
			 * The cleanup - delete_functions is called
			 * in btf_encoder__encode_cu error path.
			 */
			return -1;
		}
		encoder->functions.entries = new;
	}

	encoder->functions.entries[encoder->functions.cnt].name = name;
	if (strchr(name, '.')) {
		const char *suffix = strchr(name, '.');

		encoder->functions.suffix_cnt++;
		encoder->functions.entries[encoder->functions.cnt].prefixlen = suffix - name;
	}
	encoder->functions.entries[encoder->functions.cnt].generated = false;
	encoder->functions.entries[encoder->functions.cnt].function = NULL;
	encoder->functions.entries[encoder->functions.cnt].state.got_proto = false;
	encoder->functions.entries[encoder->functions.cnt].state.proto[0] = '\0';
	encoder->functions.entries[encoder->functions.cnt].state.type_id_off = 0;
	encoder->functions.cnt++;
	return 0;
}

static struct elf_function *btf_encoder__find_function(const struct btf_encoder *encoder,
						       const char *name, size_t prefixlen)
{
	struct elf_function key = { .name = name, .prefixlen = prefixlen };

	return bsearch(&key, encoder->functions.entries, encoder->functions.cnt, sizeof(key), functions_cmp);
}

static bool btf_name_char_ok(char c, bool first)
{
	if (c == '_' || c == '.')
		return true;

	return first ? isalpha(c) : isalnum(c);
}

/* Check whether the given name is valid in vmlinux btf. */
static bool btf_name_valid(const char *p)
{
	const char *limit;

	if (!btf_name_char_ok(*p, true))
		return false;

	/* set a limit on identifier length */
	limit = p + KSYM_NAME_LEN;
	p++;
	while (*p && p < limit) {
		if (!btf_name_char_ok(*p, false))
			return false;
		p++;
	}

	return !*p;
}

static void dump_invalid_symbol(const char *msg, const char *sym,
				int verbose, bool force)
{
	if (force) {
		if (verbose)
			fprintf(stderr, "PAHOLE: Warning: %s, ignored (sym: '%s').\n",
				msg, sym);
		return;
	}

	fprintf(stderr, "PAHOLE: Error: %s (sym: '%s').\n", msg, sym);
	fprintf(stderr, "PAHOLE: Error: Use '--btf_encode_force' to ignore such symbols and force emit the btf.\n");
}

static int tag__check_id_drift(struct btf_encoder *encoder, const struct tag *tag,
			       uint32_t core_id, uint32_t btf_type_id)
{
	if (btf_type_id != (core_id + encoder->type_id_off)) {
		fprintf(stderr,
			"%s: %s id drift, core_id: %u, btf_type_id: %u, type_id_off: %u\n",
			__func__, dwarf_tag_name(tag->tag),
			core_id, btf_type_id, encoder->type_id_off);
		return -1;
	}

	return 0;
}

static int32_t btf_encoder__add_struct_type(struct btf_encoder *encoder, struct tag *tag)
{
	struct type *type = tag__type(tag);
	struct class_member *pos;
	const char *name = type__name(type);
	int32_t type_id;
	uint8_t kind;

	kind = (tag->tag == DW_TAG_union_type) ?
		BTF_KIND_UNION : BTF_KIND_STRUCT;

	type_id = btf_encoder__add_struct(encoder, kind, name, type->size);
	if (type_id < 0)
		return type_id;

	type__for_each_data_member(type, pos) {
		/*
		 * dwarf_loader uses DWARF's recommended bit offset addressing
		 * scheme, which conforms to BTF requirement, so no conversion
		 * is required.
		 */
		name = class_member__name(pos);
		if (btf_encoder__add_field(encoder, name, encoder->type_id_off + pos->tag.type,
					   pos->bitfield_size, pos->bit_offset))
			return -1;
	}

	return type_id;
}

static uint32_t array_type__nelems(struct tag *tag)
{
	int i;
	uint32_t nelem = 1;
	struct array_type *array = tag__array_type(tag);

	for (i = array->dimensions - 1; i >= 0; --i)
		nelem *= array->nr_entries[i];

	return nelem;
}

static int32_t btf_encoder__add_enum_type(struct btf_encoder *encoder, struct tag *tag,
					  struct conf_load *conf_load)
{
	struct type *etype = tag__type(tag);
	struct enumerator *pos;
	const char *name = type__name(etype);
	int32_t type_id;

	type_id = btf_encoder__add_enum(encoder, name, etype, conf_load);
	if (type_id < 0)
		return type_id;

	type__for_each_enumerator(etype, pos) {
		name = enumerator__name(pos);
		if (btf_encoder__add_enum_val(encoder, name, pos->value, etype, conf_load))
			return -1;
	}

	return type_id;
}

static int btf_encoder__encode_tag(struct btf_encoder *encoder, struct tag *tag,
				   struct conf_load *conf_load)
{
	/* single out type 0 as it represents special type "void" */
	uint32_t ref_type_id = tag->type == 0 ? 0 : encoder->type_id_off + tag->type;
	struct base_type *bt;
	const char *name;

	switch (tag->tag) {
	case DW_TAG_base_type:
		bt   = tag__base_type(tag);
		name = __base_type__name(bt);
		return btf_encoder__add_base_type(encoder, bt, name);
	case DW_TAG_const_type:
		return btf_encoder__add_ref_type(encoder, BTF_KIND_CONST, ref_type_id, NULL, false);
	case DW_TAG_pointer_type:
		return btf_encoder__add_ref_type(encoder, BTF_KIND_PTR, ref_type_id, NULL, false);
	case DW_TAG_restrict_type:
		return btf_encoder__add_ref_type(encoder, BTF_KIND_RESTRICT, ref_type_id, NULL, false);
	case DW_TAG_volatile_type:
		return btf_encoder__add_ref_type(encoder, BTF_KIND_VOLATILE, ref_type_id, NULL, false);
	case DW_TAG_typedef:
		name = namespace__name(tag__namespace(tag));
		return btf_encoder__add_ref_type(encoder, BTF_KIND_TYPEDEF, ref_type_id, name, false);
	case DW_TAG_LLVM_annotation:
		name = tag__btf_type_tag(tag)->value;
		return btf_encoder__add_ref_type(encoder, BTF_KIND_TYPE_TAG, ref_type_id, name, false);
	case DW_TAG_structure_type:
	case DW_TAG_union_type:
	case DW_TAG_class_type:
		name = namespace__name(tag__namespace(tag));
		if (tag__type(tag)->declaration)
			return btf_encoder__add_ref_type(encoder, BTF_KIND_FWD, 0, name, tag->tag == DW_TAG_union_type);
		else
			return btf_encoder__add_struct_type(encoder, tag);
	case DW_TAG_array_type:
		/* TODO: Encode one dimension at a time. */
		encoder->need_index_type = true;
		return btf_encoder__add_array(encoder, ref_type_id, encoder->array_index_id, array_type__nelems(tag));
	case DW_TAG_enumeration_type:
		return btf_encoder__add_enum_type(encoder, tag, conf_load);
	case DW_TAG_subroutine_type:
		return btf_encoder__add_func_proto(encoder, tag__ftype(tag));
        case DW_TAG_unspecified_type:
		/* Just don't encode this for now, converting anything with this type to void (0) instead.
		 *
		 * If we end up needing to encode this, one possible hack is to do as follows, as "const void".
		 *
		 * Returning zero means we skipped encoding a DWARF type.
		 */
               // btf_encoder__add_ref_type(encoder, BTF_KIND_CONST, 0, NULL, false);
               return 0;
	default:
		fprintf(stderr, "Unsupported DW_TAG_%s(0x%x): type: 0x%x\n",
			dwarf_tag_name(tag->tag), tag->tag, ref_type_id);
		return -1;
	}
}

static int btf_encoder__write_raw_file(struct btf_encoder *encoder)
{
	const char *filename = encoder->filename;
	uint32_t raw_btf_size;
	const void *raw_btf_data;
	int fd, err;

	raw_btf_data = btf__raw_data(encoder->btf, &raw_btf_size);
	if (raw_btf_data == NULL) {
		fprintf(stderr, "%s: btf__raw_data failed!\n", __func__);
		return -1;
	}

	fd = open(filename, O_WRONLY | O_CREAT, 0640);
	if (fd < 0) {
		fprintf(stderr, "%s: Couldn't open %s for writing the raw BTF info: %s\n", __func__, filename, strerror(errno));
		return -1;
	}
	err = write(fd, raw_btf_data, raw_btf_size);
	if (err < 0)
		fprintf(stderr, "%s: Couldn't write the raw BTF info to %s: %s\n", __func__, filename, strerror(errno));

	close(fd);

	if ((uint32_t)err != raw_btf_size) {
		fprintf(stderr, "%s: Could only write %d bytes to %s of raw BTF info out of %d, aborting\n", __func__, err, filename, raw_btf_size);
		unlink(filename);
		err = -1;
	} else {
		/* go from bytes written == raw_btf_size to an indication that all went fine */
		err = 0;
	}

	return err;
}

static int btf_encoder__write_elf(struct btf_encoder *encoder)
{
	struct btf *btf = encoder->btf;
	const char *filename = encoder->filename;
	GElf_Shdr shdr_mem, *shdr;
	Elf_Data *btf_data = NULL;
	Elf_Scn *scn = NULL;
	Elf *elf = NULL;
	const void *raw_btf_data;
	uint32_t raw_btf_size;
	int fd, err = -1;
	size_t strndx;

	fd = open(filename, O_RDWR);
	if (fd < 0) {
		fprintf(stderr, "Cannot open %s\n", filename);
		return -1;
	}

	if (elf_version(EV_CURRENT) == EV_NONE) {
		elf_error("Cannot set libelf version");
		goto out;
	}

	elf = elf_begin(fd, ELF_C_RDWR, NULL);
	if (elf == NULL) {
		elf_error("Cannot update ELF file");
		goto out;
	}

	elf_flagelf(elf, ELF_C_SET, ELF_F_DIRTY);

	/*
	 * First we look if there was already a .BTF section to overwrite.
	 */

	elf_getshdrstrndx(elf, &strndx);
	while ((scn = elf_nextscn(elf, scn)) != NULL) {
		shdr = gelf_getshdr(scn, &shdr_mem);
		if (shdr == NULL)
			continue;
		char *secname = elf_strptr(elf, strndx, shdr->sh_name);
		if (strcmp(secname, ".BTF") == 0) {
			btf_data = elf_getdata(scn, btf_data);
			break;
		}
	}

	raw_btf_data = btf__raw_data(btf, &raw_btf_size);

	if (btf_data) {
		/* Existing .BTF section found */
		btf_data->d_buf = (void *)raw_btf_data;
		btf_data->d_size = raw_btf_size;
		elf_flagdata(btf_data, ELF_C_SET, ELF_F_DIRTY);

		if (elf_update(elf, ELF_C_NULL) >= 0 &&
		    elf_update(elf, ELF_C_WRITE) >= 0)
			err = 0;
		else
			elf_error("elf_update failed");
	} else {
		const char *llvm_objcopy;
		char tmp_fn[PATH_MAX];
		char cmd[PATH_MAX * 2];

		llvm_objcopy = getenv("LLVM_OBJCOPY");
		if (!llvm_objcopy)
			llvm_objcopy = "llvm-objcopy";

		/* Use objcopy to add a .BTF section */
		snprintf(tmp_fn, sizeof(tmp_fn), "%s.btf", filename);
		close(fd);
		fd = creat(tmp_fn, S_IRUSR | S_IWUSR);
		if (fd == -1) {
			fprintf(stderr, "%s: open(%s) failed!\n", __func__,
				tmp_fn);
			goto out;
		}

		if (write(fd, raw_btf_data, raw_btf_size) != raw_btf_size) {
			fprintf(stderr, "%s: write of %d bytes to '%s' failed: %d!\n",
				__func__, raw_btf_size, tmp_fn, errno);
			goto unlink;
		}

		snprintf(cmd, sizeof(cmd), "%s --add-section .BTF=%s %s",
			 llvm_objcopy, tmp_fn, filename);
		if (system(cmd)) {
			fprintf(stderr, "%s: failed to add .BTF section to '%s': %d!\n",
				__func__, filename, errno);
			goto unlink;
		}

		err = 0;
	unlink:
		unlink(tmp_fn);
	}

out:
	if (fd != -1)
		close(fd);
	if (elf)
		elf_end(elf);
	return err;
}

int btf_encoder__encode(struct btf_encoder *encoder)
{
	int err;

	/* for single-threaded case, saved funcs are added here */
	btf_encoder__add_saved_funcs(encoder);

	if (gobuffer__size(&encoder->percpu_secinfo) != 0)
		btf_encoder__add_datasec(encoder, PERCPU_SECTION);

	/* Empty file, nothing to do, so... done! */
	if (btf__type_cnt(encoder->btf) == 1)
		return 0;

	if (btf__dedup(encoder->btf, NULL)) {
		fprintf(stderr, "%s: btf__dedup failed!\n", __func__);
		return -1;
	}

	if (encoder->raw_output)
		err = btf_encoder__write_raw_file(encoder);
	else
		err = btf_encoder__write_elf(encoder);

	return err;
}

static int percpu_var_cmp(const void *_a, const void *_b)
{
	const struct var_info *a = _a;
	const struct var_info *b = _b;

	if (a->addr == b->addr)
		return 0;
	return a->addr < b->addr ? -1 : 1;
}

static bool btf_encoder__percpu_var_exists(struct btf_encoder *encoder, uint64_t addr, uint32_t *sz, const char **name)
{
	struct var_info key = { .addr = addr };
	const struct var_info *p = bsearch(&key, encoder->percpu.vars, encoder->percpu.var_cnt,
					   sizeof(encoder->percpu.vars[0]), percpu_var_cmp);
	if (!p)
		return false;

	*sz = p->sz;
	*name = p->name;
	return true;
}

static int btf_encoder__collect_percpu_var(struct btf_encoder *encoder, GElf_Sym *sym, size_t sym_sec_idx)
{
	const char *sym_name;
	uint64_t addr;
	uint32_t size;

	/* compare a symbol's shndx to determine if it's a percpu variable */
	if (sym_sec_idx != encoder->percpu.shndx)
		return 0;
	if (elf_sym__type(sym) != STT_OBJECT)
		return 0;

	addr = elf_sym__value(sym);

	size = elf_sym__size(sym);
	if (!size)
		return 0; /* ignore zero-sized symbols */

	sym_name = elf_sym__name(sym, encoder->symtab);
	if (!btf_name_valid(sym_name)) {
		dump_invalid_symbol("Found symbol of invalid name when encoding btf",
				    sym_name, encoder->verbose, encoder->force);
		if (encoder->force)
			return 0;
		return -1;
	}

	if (encoder->verbose)
		printf("Found per-CPU symbol '%s' at address 0x%" PRIx64 "\n", sym_name, addr);

	/* Make sure addr is section-relative. For kernel modules (which are
	 * ET_REL files) this is already the case. For vmlinux (which is an
	 * ET_EXEC file) we need to subtract the section address.
	 */
	if (!encoder->is_rel)
		addr -= encoder->percpu.base_addr;

	if (encoder->percpu.var_cnt == MAX_PERCPU_VAR_CNT) {
		fprintf(stderr, "Reached the limit of per-CPU variables: %d\n",
			MAX_PERCPU_VAR_CNT);
		return -1;
	}
	encoder->percpu.vars[encoder->percpu.var_cnt].addr = addr;
	encoder->percpu.vars[encoder->percpu.var_cnt].sz = size;
	encoder->percpu.vars[encoder->percpu.var_cnt].name = sym_name;
	encoder->percpu.var_cnt++;

	return 0;
}

static int btf_encoder__collect_symbols(struct btf_encoder *encoder, bool collect_percpu_vars)
{
	Elf32_Word sym_sec_idx;
	uint32_t core_id;
	GElf_Sym sym;

	/* cache variables' addresses, preparing for searching in symtab. */
	encoder->percpu.var_cnt = 0;

	/* search within symtab for percpu variables */
	elf_symtab__for_each_symbol_index(encoder->symtab, core_id, sym, sym_sec_idx) {
		if (collect_percpu_vars && btf_encoder__collect_percpu_var(encoder, &sym, sym_sec_idx))
			return -1;
		if (btf_encoder__collect_function(encoder, &sym))
			return -1;
	}

	if (collect_percpu_vars) {
		if (encoder->percpu.var_cnt)
			qsort(encoder->percpu.vars, encoder->percpu.var_cnt, sizeof(encoder->percpu.vars[0]), percpu_var_cmp);

		if (encoder->verbose)
			printf("Found %d per-CPU variables!\n", encoder->percpu.var_cnt);
	}

	if (encoder->functions.cnt) {
		qsort(encoder->functions.entries, encoder->functions.cnt, sizeof(encoder->functions.entries[0]),
		      functions_cmp);
		if (encoder->verbose)
			printf("Found %d functions!\n", encoder->functions.cnt);
	}

	return 0;
}

static bool ftype__has_arg_names(const struct ftype *ftype)
{
	struct parameter *param;

	ftype__for_each_parameter(ftype, param) {
		if (parameter__name(param) == NULL)
			return false;
	}
	return true;
}

static int btf_encoder__encode_cu_variables(struct btf_encoder *encoder)
{
	struct cu *cu = encoder->cu;
	uint32_t core_id;
	struct tag *pos;
	int err = -1;

	if (encoder->percpu.shndx == 0 || !encoder->symtab)
		return 0;

	if (encoder->verbose)
		printf("search cu '%s' for percpu global variables.\n", cu->name);

	cu__for_each_variable(cu, core_id, pos) {
		struct variable *var = tag__variable(pos);
		uint32_t size, type, linkage;
		const char *name, *dwarf_name;
		struct llvm_annotation *annot;
		const struct tag *tag;
		uint64_t addr;
		int id;

		if (var->declaration && !var->spec)
			continue;

		/* percpu variables are allocated in global space */
		if (variable__scope(var) != VSCOPE_GLOBAL && !var->spec)
			continue;

		/* addr has to be recorded before we follow spec */
		addr = var->ip.addr;
		dwarf_name = variable__name(var);

		/* Make sure addr is section-relative. DWARF, unlike ELF,
		 * always contains virtual symbol addresses, so subtract
		 * the section address unconditionally.
		 */
		if (addr < encoder->percpu.base_addr || addr >= encoder->percpu.base_addr + encoder->percpu.sec_sz)
			continue;
		addr -= encoder->percpu.base_addr;

		if (!btf_encoder__percpu_var_exists(encoder, addr, &size, &name))
			continue; /* not a per-CPU variable */

		/* A lot of "special" DWARF variables (e.g, __UNIQUE_ID___xxx)
		 * have addr == 0, which is the same as, say, valid
		 * fixed_percpu_data per-CPU variable. To distinguish between
		 * them, additionally compare DWARF and ELF symbol names. If
		 * DWARF doesn't provide proper name, pessimistically assume
		 * bad variable.
		 *
		 * Examples of such special variables are:
		 *
		 *  1. __ADDRESSABLE(sym), which are forcely emitted as symbols.
		 *  2. __UNIQUE_ID(prefix), which are introduced to generate unique ids.
		 *  3. __exitcall(fn), functions which are labeled as exit calls.
		 *
		 *  This is relevant only for vmlinux image, as for kernel
		 *  modules per-CPU data section has non-zero offset so all
		 *  per-CPU symbols have non-zero values.
		 */
		if (var->ip.addr == 0) {
			if (!dwarf_name || strcmp(dwarf_name, name))
				continue;
		}

		if (var->spec)
			var = var->spec;

		if (var->ip.tag.type == 0) {
			fprintf(stderr, "error: found variable '%s' in CU '%s' that has void type\n",
				name, cu->name);
			if (encoder->force)
				continue;
			err = -1;
			break;
		}

		tag = cu__type(cu, var->ip.tag.type);
		if (tag__size(tag, cu) == 0) {
			if (encoder->verbose)
				fprintf(stderr, "Ignoring zero-sized per-CPU variable '%s'...\n", dwarf_name ?: "<missing name>");
			continue;
		}

		type = var->ip.tag.type + encoder->type_id_off;
		linkage = var->external ? BTF_VAR_GLOBAL_ALLOCATED : BTF_VAR_STATIC;

		if (encoder->verbose) {
			printf("Variable '%s' from CU '%s' at address 0x%" PRIx64 " encoded\n",
			       name, cu->name, addr);
		}

		/* add a BTF_KIND_VAR in encoder->types */
		id = btf_encoder__add_var(encoder, type, name, linkage);
		if (id < 0) {
			fprintf(stderr, "error: failed to encode variable '%s' at addr 0x%" PRIx64 "\n",
			        name, addr);
			goto out;
		}

		list_for_each_entry(annot, &var->annots, node) {
			int tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, id, annot->component_idx);
			if (tag_type_id < 0) {
				fprintf(stderr, "error: failed to encode tag '%s' to variable '%s' with component_idx %d\n",
					annot->value, name, annot->component_idx);
				goto out;
			}
		}

		/*
		 * add a BTF_VAR_SECINFO in encoder->percpu_secinfo, which will be added into
		 * encoder->types later when we add BTF_VAR_DATASEC.
		 */
		id = btf_encoder__add_var_secinfo(encoder, id, addr, size);
		if (id < 0) {
			fprintf(stderr, "error: failed to encode section info for variable '%s' at addr 0x%" PRIx64 "\n",
			        name, addr);
			goto out;
		}
	}

	err = 0;
out:
	return err;
}

struct btf_encoder *btf_encoder__new(struct cu *cu, const char *detached_filename, struct btf *base_btf, bool skip_encoding_vars, bool force, bool gen_floats, bool verbose)
{
	struct btf_encoder *encoder = zalloc(sizeof(*encoder));

	if (encoder) {
		encoder->raw_output = detached_filename != NULL;
		encoder->filename = strdup(encoder->raw_output ? detached_filename : cu->filename);
		if (encoder->filename == NULL)
			goto out_delete;

		encoder->btf = btf__new_empty_split(base_btf);
		if (encoder->btf == NULL)
			goto out_delete;

		encoder->force		 = force;
		encoder->gen_floats	 = gen_floats;
		encoder->skip_encoding_vars = skip_encoding_vars;
		encoder->verbose	 = verbose;
		encoder->has_index_type  = false;
		encoder->need_index_type = false;
		encoder->array_index_id  = 0;

		GElf_Ehdr ehdr;

		if (gelf_getehdr(cu->elf, &ehdr) == NULL) {
			if (encoder->verbose)
				elf_error("cannot get ELF header");
			goto out_delete;
		}

		encoder->is_rel = ehdr.e_type == ET_REL;

		switch (ehdr.e_ident[EI_DATA]) {
		case ELFDATA2LSB:
			btf__set_endianness(encoder->btf, BTF_LITTLE_ENDIAN);
			break;
		case ELFDATA2MSB:
			btf__set_endianness(encoder->btf, BTF_BIG_ENDIAN);
			break;
		default:
			fprintf(stderr, "%s: unknown ELF endianness.\n", __func__);
			goto out_delete;
		}

		encoder->symtab = elf_symtab__new(NULL, cu->elf);
		if (!encoder->symtab) {
			if (encoder->verbose)
				printf("%s: '%s' doesn't have symtab.\n", __func__, cu->filename);
			goto out;
		}

		/* find percpu section's shndx */

		GElf_Shdr shdr;
		Elf_Scn *sec = elf_section_by_name(cu->elf, &shdr, PERCPU_SECTION, NULL);

		if (!sec) {
			if (encoder->verbose)
				printf("%s: '%s' doesn't have '%s' section\n", __func__, cu->filename, PERCPU_SECTION);
		} else {
			encoder->percpu.shndx	  = elf_ndxscn(sec);
			encoder->percpu.base_addr = shdr.sh_addr;
			encoder->percpu.sec_sz	  = shdr.sh_size;
		}

		if (btf_encoder__collect_symbols(encoder, !encoder->skip_encoding_vars))
			goto out_delete;

		if (encoder->verbose)
			printf("File %s:\n", cu->filename);
		btf_encoders__add(encoder);
	}
out:
	return encoder;

out_delete:
	btf_encoder__delete(encoder);
	return NULL;
}

void btf_encoder__delete(struct btf_encoder *encoder)
{
	if (encoder == NULL)
		return;

	btf_encoders__delete(encoder);
	__gobuffer__delete(&encoder->percpu_secinfo);
	zfree(&encoder->filename);
	btf__free(encoder->btf);
	encoder->btf = NULL;
	elf_symtab__delete(encoder->symtab);

	encoder->functions.allocated = encoder->functions.cnt = 0;
	free(encoder->functions.entries);
	encoder->functions.entries = NULL;

	free(encoder);
}

int btf_encoder__encode_cu(struct btf_encoder *encoder, struct cu *cu, struct conf_load *conf_load)
{
	struct llvm_annotation *annot;
	int btf_type_id, tag_type_id, skipped_types = 0;
	uint32_t core_id;
	struct function *fn;
	struct tag *pos;
	int err = 0;

	encoder->cu = cu;
	encoder->type_id_off = btf__type_cnt(encoder->btf) - 1;

	if (!encoder->has_index_type) {
		/* cu__find_base_type_by_name() takes "type_id_t *id" */
		type_id_t id;
		if (cu__find_base_type_by_name(cu, "int", &id)) {
			encoder->has_index_type = true;
			encoder->array_index_id = encoder->type_id_off + id;
		} else {
			encoder->has_index_type = false;
			encoder->array_index_id = encoder->type_id_off + cu->types_table.nr_entries;
		}
	}

	cu__for_each_type(cu, core_id, pos) {
		btf_type_id = btf_encoder__encode_tag(encoder, pos, conf_load);

		if (btf_type_id == 0) {
			++skipped_types;
			continue;
		}

		if (btf_type_id < 0 ||
		    tag__check_id_drift(encoder, pos, core_id, btf_type_id + skipped_types)) {
			err = -1;
			goto out;
		}
	}

	if (encoder->need_index_type && !encoder->has_index_type) {
		struct base_type bt = {};

		bt.name = 0;
		bt.bit_size = 32;
		bt.is_signed = true;
		btf_encoder__add_base_type(encoder, &bt, "int");
		encoder->has_index_type = true;
	}

	cu__for_each_type(cu, core_id, pos) {
		struct namespace *ns;
		const char *tag_name;

		switch (pos->tag) {
		case DW_TAG_structure_type:
			tag_name = "struct";
			break;
		case DW_TAG_union_type:
			tag_name = "union";
			break;
		case DW_TAG_typedef:
			tag_name = "typedef";
			break;
		default:
			continue;
		}

		btf_type_id = encoder->type_id_off + core_id;
		ns = tag__namespace(pos);
		list_for_each_entry(annot, &ns->annots, node) {
			tag_type_id = btf_encoder__add_decl_tag(encoder, annot->value, btf_type_id, annot->component_idx);
			if (tag_type_id < 0) {
				fprintf(stderr, "error: failed to encode tag '%s' to %s '%s' with component_idx %d\n",
					annot->value, tag_name, namespace__name(ns), annot->component_idx);
				goto out;
			}
		}
	}

	cu__for_each_function(cu, core_id, fn) {
		struct elf_function *func = NULL;
		bool save = false;

		/*
		 * Skip functions that:
		 *   - are marked as declarations
		 *   - do not have full argument names
		 *   - are not in ftrace list (if it's available)
		 *   - are not external (in case ftrace filter is not available)
		 */
		if (fn->declaration)
			continue;
		if (!ftype__has_arg_names(&fn->proto))
			continue;
		if (encoder->functions.cnt) {
			const char *name;

			name = function__name(fn);
			if (!name)
				continue;

			/* prefer exact function name match... */
			func = btf_encoder__find_function(encoder, name, 0);
			if (func) {
				if (func->generated)
					continue;
				if (conf_load->skip_encoding_btf_inconsistent_proto)
					save = true;
				else
					func->generated = true;
			} else if (encoder->functions.suffix_cnt &&
				   conf_load->btf_gen_optimized) {
				/* falling back to name.isra.0 match if no exact
				 * match is found; only bother if we found any
				 * .suffix function names.  The function
				 * will be saved and added once we ensure
				 * it does not have optimized-out parameters
				 * in any cu.
				 */
				func = btf_encoder__find_function(encoder, name,
								  strlen(name));
				if (func) {
					save = true;
					if (encoder->verbose)
						printf("matched function '%s' with '%s'%s\n",
						       name, func->name,
						       fn->proto.optimized_parms ?
						       ", has optimized-out parameters" :
						       fn->proto.unexpected_reg ? ", has unexpected register use by params" :
						       "");
					fn->alias = func->name;
				}
			}
			if (!func)
				continue;
		} else {
			if (!fn->external)
				continue;
		}

		if (save)
			err = btf_encoder__save_func(encoder, fn, func);
		else
			err = btf_encoder__add_func(encoder, fn);
		if (err)
			goto out;
	}

	if (!encoder->skip_encoding_vars)
		err = btf_encoder__encode_cu_variables(encoder);

	/* It is only safe to delete this CU if we have not stashed any static
	 * functions for later addition.
	 */
	if (!err)
		err = encoder->saved_func_cnt > 0 ? LSK__KEEPIT : LSK__DELETE;
out:
	encoder->cu = NULL;
	return err;
}

struct btf *btf_encoder__btf(struct btf_encoder *encoder)
{
	return encoder->btf;
}