arm64: Add a data+tag clean+invalidate primitive
Signed-off-by: Marc Zyngier <maz@kernel.org>
diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h
index 37185e9..15f021c 100644
--- a/arch/arm64/include/asm/cacheflush.h
+++ b/arch/arm64/include/asm/cacheflush.h
@@ -72,6 +72,7 @@
extern void caches_clean_inval_pou(unsigned long start, unsigned long end);
extern void icache_inval_pou(unsigned long start, unsigned long end);
extern void dcache_clean_inval_poc(unsigned long start, unsigned long end);
+extern void dcache_tag_clean_inval_poc(unsigned long start, unsigned long end);
extern void dcache_inval_poc(unsigned long start, unsigned long end);
extern void dcache_clean_poc(unsigned long start, unsigned long end);
extern void dcache_clean_pop(unsigned long start, unsigned long end);
diff --git a/arch/arm64/kvm/hyp/nvhe/cache.S b/arch/arm64/kvm/hyp/nvhe/cache.S
index 85936c1..1863d5a 100644
--- a/arch/arm64/kvm/hyp/nvhe/cache.S
+++ b/arch/arm64/kvm/hyp/nvhe/cache.S
@@ -13,6 +13,17 @@
SYM_FUNC_END(__pi_dcache_clean_inval_poc)
SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc)
+SYM_FUNC_START(__pi_dcache_tag_clean_inval_poc)
+#ifdef CONFIG_ARM64_MTE
+ .arch_extension memtag
+ dcache_by_line_op cigdvac, sy, x0, x1, x2, x3
+#else
+ ASM_BUG()
+#endif
+ ret
+SYM_FUNC_END(__pi_dcache_tag_clean_inval_poc)
+SYM_FUNC_ALIAS(dcache_tag_clean_inval_poc, __pi_dcache_tag_clean_inval_poc)
+
SYM_FUNC_START(__pi_icache_inval_pou)
alternative_if ARM64_HAS_CACHE_DIC
isb
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 503567c..b495b8b 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -115,6 +115,26 @@
SYM_FUNC_ALIAS(dcache_clean_inval_poc, __pi_dcache_clean_inval_poc)
/*
+ * dcache_tag_clean_inval_poc(start, end)
+ *
+ * Ensure that any D-cache lines and MTE tags for the interval
+ * [start, end) are cleaned and invalidated to the PoC.
+ *
+ * - start - virtual start address of region
+ * - end - virtual end address of region
+ */
+SYM_FUNC_START(__pi_dcache_tag_clean_inval_poc)
+#ifdef CONFIG_ARM64_MTE
+ .arch_extension memtag
+ dcache_by_line_op cigdvac, sy, x0, x1, x2, x3
+#else
+ ASM_BUG()
+#endif
+ ret
+SYM_FUNC_END(__pi_dcache_tag_clean_inval_poc)
+SYM_FUNC_ALIAS(dcache_tag_clean_inval_poc, __pi_dcache_tag_clean_inval_poc)
+
+/*
* dcache_clean_pou(start, end)
*
* Ensure that any D-cache lines for the interval [start, end)
