1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
From 335c32d2653f2415e042c8ad1ae2a6b33019b96f Mon Sep 17 00:00:00 2001
From: Emil Renner Berthing <kernel@esmil.dk>
Date: Sat, 12 Jun 2021 16:48:31 -0700
Subject: [PATCH 1019/1024] soc: sifive: ccache: Add non-coherent DMA handling
Add functions to flush the caches and handle non-coherent DMA.
Signed-off-by: Emil Renner Berthing <kernel@esmil.dk>
---
drivers/soc/sifive/sifive_ccache.c | 60 +++++++++++++++++++++++++++++-
include/soc/sifive/sifive_ccache.h | 21 +++++++++++
2 files changed, 80 insertions(+), 1 deletion(-)
--- a/drivers/soc/sifive/sifive_ccache.c
+++ b/drivers/soc/sifive/sifive_ccache.c
@@ -8,13 +8,16 @@
#define pr_fmt(fmt) "CCACHE: " fmt
+#include <linux/align.h>
#include <linux/debugfs.h>
#include <linux/interrupt.h>
#include <linux/of_irq.h>
#include <linux/of_address.h>
#include <linux/device.h>
#include <linux/bitfield.h>
+#include <asm/cacheflush.h>
#include <asm/cacheinfo.h>
+#include <asm/page.h>
#include <soc/sifive/sifive_ccache.h>
#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100
@@ -39,10 +42,14 @@
#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16)
#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24)
+#define SIFIVE_CCACHE_FLUSH64 0x200
+#define SIFIVE_CCACHE_FLUSH32 0x240
+
#define SIFIVE_CCACHE_WAYENABLE 0x08
#define SIFIVE_CCACHE_ECCINJECTERR 0x40
#define SIFIVE_CCACHE_MAX_ECCINTR 4
+#define SIFIVE_CCACHE_LINE_SIZE 64
static void __iomem *ccache_base;
static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];
@@ -126,6 +133,47 @@ int unregister_sifive_ccache_error_notif
}
EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier);
+#ifdef CONFIG_RISCV_DMA_NONCOHERENT
+static phys_addr_t uncached_offset;
+DEFINE_STATIC_KEY_FALSE(sifive_ccache_handle_noncoherent_key);
+
+void sifive_ccache_flush_range(phys_addr_t start, size_t len)
+{
+ phys_addr_t end = start + len;
+ phys_addr_t line;
+
+ if (!len)
+ return;
+
+ mb();
+ for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end;
+ line += SIFIVE_CCACHE_LINE_SIZE) {
+#ifdef CONFIG_32BIT
+ writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32);
+#else
+ writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64);
+#endif
+ mb();
+ }
+}
+EXPORT_SYMBOL_GPL(sifive_ccache_flush_range);
+
+void *sifive_ccache_set_uncached(void *addr, size_t size)
+{
+ phys_addr_t phys_addr = __pa(addr) + uncached_offset;
+ void *mem_base;
+
+ mem_base = memremap(phys_addr, size, MEMREMAP_WT);
+ if (!mem_base) {
+ pr_err("%s memremap failed for addr %p\n", __func__, addr);
+ return ERR_PTR(-EINVAL);
+ }
+
+ return mem_base;
+}
+EXPORT_SYMBOL_GPL(sifive_ccache_set_uncached);
+#endif /* CONFIG_RISCV_DMA_NONCOHERENT */
+
static int ccache_largest_wayenabled(void)
{
return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF;
@@ -214,6 +262,7 @@ static int __init sifive_ccache_init(voi
int i, rc, intr_num;
const struct of_device_id *match;
unsigned long broken_irqs;
+ u64 __maybe_unused offset;
np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match);
if (!np)
@@ -259,6 +308,15 @@ static int __init sifive_ccache_init(voi
}
of_node_put(np);
+#ifdef CONFIG_RISCV_DMA_NONCOHERENT
+ if (!of_property_read_u64(np, "uncached-offset", &offset)) {
+ uncached_offset = offset;
+ static_branch_enable(&sifive_ccache_handle_noncoherent_key);
+ riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE;
+ riscv_noncoherent_supported();
+ }
+#endif
+
ccache_config_read();
ccache_cache_ops.get_priv_group = ccache_get_priv_group;
@@ -279,4 +337,4 @@ err_node_put:
return rc;
}
-device_initcall(sifive_ccache_init);
+arch_initcall(sifive_ccache_init);
--- a/include/soc/sifive/sifive_ccache.h
+++ b/include/soc/sifive/sifive_ccache.h
@@ -7,10 +7,31 @@
#ifndef __SOC_SIFIVE_CCACHE_H
#define __SOC_SIFIVE_CCACHE_H
+#include <linux/io.h>
+#include <linux/jump_label.h>
+
extern int register_sifive_ccache_error_notifier(struct notifier_block *nb);
extern int unregister_sifive_ccache_error_notifier(struct notifier_block *nb);
#define SIFIVE_CCACHE_ERR_TYPE_CE 0
#define SIFIVE_CCACHE_ERR_TYPE_UE 1
+DECLARE_STATIC_KEY_FALSE(sifive_ccache_handle_noncoherent_key);
+
+static inline bool sifive_ccache_handle_noncoherent(void)
+{
+#ifdef CONFIG_SIFIVE_CCACHE
+ return static_branch_unlikely(&sifive_ccache_handle_noncoherent_key);
+#else
+ return false;
+#endif
+}
+
+void sifive_ccache_flush_range(phys_addr_t start, size_t len);
+void *sifive_ccache_set_uncached(void *addr, size_t size);
+static inline void sifive_ccache_clear_uncached(void *addr, size_t size)
+{
+ memunmap(addr);
+}
+
#endif /* __SOC_SIFIVE_CCACHE_H */
|