1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2018, The Linux Foundation. All rights reserved. */
/* Copyright (c) 2023 Collabora, Ltd. */
/* Copyright (c) 2024 Valve Corporation */
/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
/*
* Try to transition the preemption state from old to new. Return
* true on success or false if the original state wasn't 'old'
*/
static inline bool try_preempt_state(struct a6xx_gpu *a6xx_gpu,
enum a6xx_preempt_state old, enum a6xx_preempt_state new)
{
enum a6xx_preempt_state cur = atomic_cmpxchg(&a6xx_gpu->preempt_state,
old, new);
return (cur == old);
}
/*
* Force the preemption state to the specified state. This is used in cases
* where the current state is known and won't change
*/
static inline void set_preempt_state(struct a6xx_gpu *gpu,
enum a6xx_preempt_state new)
{
/*
* preempt_state may be read by other cores trying to trigger a
* preemption or in the interrupt handler so barriers are needed
* before...
*/
smp_mb__before_atomic();
atomic_set(&gpu->preempt_state, new);
/* ... and after */
smp_mb__after_atomic();
}
/* Write the most recent wptr for the given ring into the hardware */
static inline void update_wptr(struct a6xx_gpu *a6xx_gpu, struct msm_ringbuffer *ring)
{
unsigned long flags;
uint32_t wptr;
spin_lock_irqsave(&ring->preempt_lock, flags);
if (ring->restore_wptr) {
wptr = get_wptr(ring);
a6xx_fenced_write(a6xx_gpu, REG_A6XX_CP_RB_WPTR, wptr, BIT(0), false);
ring->restore_wptr = false;
}
spin_unlock_irqrestore(&ring->preempt_lock, flags);
}
/* Return the highest priority ringbuffer with something in it */
static inline struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
{
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
unsigned long flags;
int i;
for (i = 0; i < gpu->nr_rings; i++) {
bool empty;
struct msm_ringbuffer *ring = gpu->rb[i];
spin_lock_irqsave(&ring->preempt_lock, flags);
empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
if (!empty && ring == a6xx_gpu->cur_ring)
empty = ring->memptrs->fence == a6xx_gpu->last_seqno[i];
spin_unlock_irqrestore(&ring->preempt_lock, flags);
if (!empty)
return ring;
}
return NULL;
}
|
