/
6x8c4-minmax-fp32-neondot.c
303 lines (277 loc) · 15.7 KB
/
6x8c4-minmax-fp32-neondot.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
// Auto-generated file. Do not edit!
// Template: src/qs8-gemm/MRxNRc4-neondot.c.in
// Generator: tools/xngen
//
// Copyright 2020 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
#include <assert.h>
#include <arm_neon.h>
#include <xnnpack/gemm.h>
#include <xnnpack/intrinsics-polyfill.h>
#include <xnnpack/math.h>
void xnn_qs8_gemm_minmax_fp32_ukernel_6x8c4__neondot(
size_t mr,
size_t nc,
size_t kc,
const int8_t* restrict a,
size_t a_stride,
const void* restrict w,
int8_t* restrict c,
size_t cm_stride,
size_t cn_stride,
const union xnn_qs8_conv_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN XNN_DISABLE_MSAN
{
assert(mr != 0);
assert(mr <= 6);
assert(nc != 0);
assert(kc != 0);
assert(kc % sizeof(int8_t) == 0);
assert(a != NULL);
assert(w != NULL);
assert(c != NULL);
kc = round_up_po2(kc, 4);
const int8_t* a0 = a;
int8_t* c0 = c;
const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride);
int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride);
if XNN_UNPREDICTABLE(mr < 2) {
a1 = a0;
c1 = c0;
}
const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride);
int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride);
if XNN_UNPREDICTABLE(mr <= 2) {
a2 = a1;
c2 = c1;
}
const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride);
int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride);
if XNN_UNPREDICTABLE(mr < 4) {
a3 = a2;
c3 = c2;
}
const int8_t* a4 = (const int8_t*) ((uintptr_t) a3 + a_stride);
int8_t* c4 = (int8_t*) ((uintptr_t) c3 + cm_stride);
if XNN_UNPREDICTABLE(mr <= 4) {
a4 = a3;
c4 = c3;
}
const int8_t* a5 = (const int8_t*) ((uintptr_t) a4 + a_stride);
int8_t* c5 = (int8_t*) ((uintptr_t) c4 + cm_stride);
if XNN_UNPREDICTABLE(mr != 6) {
a5 = a4;
c5 = c4;
}
// Loop over groups of 8 columns.
do {
// Initialize accumulators with bias. 8 bias values are loaded from the
// weight matrix, at the start of the group of 8 columns.
int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t));
int32x4_t vacc1x0123 = vacc0x0123;
int32x4_t vacc1x4567 = vacc0x4567;
int32x4_t vacc2x0123 = vacc0x0123;
int32x4_t vacc2x4567 = vacc0x4567;
int32x4_t vacc3x0123 = vacc0x0123;
int32x4_t vacc3x4567 = vacc0x4567;
int32x4_t vacc4x0123 = vacc0x0123;
int32x4_t vacc4x4567 = vacc0x4567;
int32x4_t vacc5x0123 = vacc0x0123;
int32x4_t vacc5x4567 = vacc0x4567;
// Inner accumulation loop along the 8 columns.
size_t k = kc;
// 2x partial unrolled loop to load 8 bytes at a time.
while (k >= 8 * sizeof(int8_t)) {
// Load a 6x8 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0); a0 += 8;
const int8x8_t va1x01234567 = vld1_s8(a1); a1 += 8;
const int8x8_t va2x01234567 = vld1_s8(a2); a2 += 8;
const int8x8_t va3x01234567 = vld1_s8(a3); a3 += 8;
const int8x8_t va4x01234567 = vld1_s8(a4); a4 += 8;
const int8x8_t va5x01234567 = vld1_s8(a5); a5 += 8;
// Load a 8x8 block of weights.
const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
const int8x16_t vb4567x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
const int8x16_t vb4567x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
// Multiply-accumulate: 6x8 * 8x8 --> 6x8.
vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0);
vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0);
vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0);
vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0);
vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0);
vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0);
vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0);
vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0);
vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0);
vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0);
vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0);
vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0);
vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb4567x0123, va0x01234567, 1);
vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb4567x4567, va0x01234567, 1);
vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb4567x0123, va1x01234567, 1);
vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb4567x4567, va1x01234567, 1);
vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb4567x0123, va2x01234567, 1);
vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb4567x4567, va2x01234567, 1);
vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb4567x0123, va3x01234567, 1);
vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb4567x4567, va3x01234567, 1);
vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb4567x0123, va4x01234567, 1);
vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb4567x4567, va4x01234567, 1);
vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb4567x0123, va5x01234567, 1);
vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb4567x4567, va5x01234567, 1);
k -= 8 * sizeof(int8_t);
}
// Handle up to 4 final positions of `k`
if XNN_UNLIKELY(k != 0) {
// Load a 6x4 block of activations.
const int8x8_t va0x01234567 = vld1_s8(a0); a0 += 4;
const int8x8_t va1x01234567 = vld1_s8(a1); a1 += 4;
const int8x8_t va2x01234567 = vld1_s8(a2); a2 += 4;
const int8x8_t va3x01234567 = vld1_s8(a3); a3 += 4;
const int8x8_t va4x01234567 = vld1_s8(a4); a4 += 4;
const int8x8_t va5x01234567 = vld1_s8(a5); a5 += 4;
// Load a 4x8 block of weights.
const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16);
// Multiply-accumulate: 6x4 * 4x8 --> 6x8.
vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0);
vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0);
vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0);
vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0);
vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0);
vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0);
vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0);
vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0);
vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0);
vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0);
vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0);
vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0);
}
// Post-accumulation work
const float32x4_t vscale = vld1q_dup_f32(¶ms->fp32_neonv8.scale);
float32x4_t vproduct0x0123 = vcvtq_f32_s32(vacc0x0123);
float32x4_t vproduct0x4567 = vcvtq_f32_s32(vacc0x4567);
float32x4_t vproduct1x0123 = vcvtq_f32_s32(vacc1x0123);
float32x4_t vproduct1x4567 = vcvtq_f32_s32(vacc1x4567);
float32x4_t vproduct2x0123 = vcvtq_f32_s32(vacc2x0123);
float32x4_t vproduct2x4567 = vcvtq_f32_s32(vacc2x4567);
float32x4_t vproduct3x0123 = vcvtq_f32_s32(vacc3x0123);
float32x4_t vproduct3x4567 = vcvtq_f32_s32(vacc3x4567);
float32x4_t vproduct4x0123 = vcvtq_f32_s32(vacc4x0123);
float32x4_t vproduct4x4567 = vcvtq_f32_s32(vacc4x4567);
float32x4_t vproduct5x0123 = vcvtq_f32_s32(vacc5x0123);
float32x4_t vproduct5x4567 = vcvtq_f32_s32(vacc5x4567);
vproduct0x0123 = vmulq_f32(vproduct0x0123, vscale);
vproduct0x4567 = vmulq_f32(vproduct0x4567, vscale);
vproduct1x0123 = vmulq_f32(vproduct1x0123, vscale);
vproduct1x4567 = vmulq_f32(vproduct1x4567, vscale);
vproduct2x0123 = vmulq_f32(vproduct2x0123, vscale);
vproduct2x4567 = vmulq_f32(vproduct2x4567, vscale);
vproduct3x0123 = vmulq_f32(vproduct3x0123, vscale);
vproduct3x4567 = vmulq_f32(vproduct3x4567, vscale);
vproduct4x0123 = vmulq_f32(vproduct4x0123, vscale);
vproduct4x4567 = vmulq_f32(vproduct4x4567, vscale);
vproduct5x0123 = vmulq_f32(vproduct5x0123, vscale);
vproduct5x4567 = vmulq_f32(vproduct5x4567, vscale);
vacc0x0123 = vcvtnq_s32_f32(vproduct0x0123);
vacc0x4567 = vcvtnq_s32_f32(vproduct0x4567);
vacc1x0123 = vcvtnq_s32_f32(vproduct1x0123);
vacc1x4567 = vcvtnq_s32_f32(vproduct1x4567);
vacc2x0123 = vcvtnq_s32_f32(vproduct2x0123);
vacc2x4567 = vcvtnq_s32_f32(vproduct2x4567);
vacc3x0123 = vcvtnq_s32_f32(vproduct3x0123);
vacc3x4567 = vcvtnq_s32_f32(vproduct3x4567);
vacc4x0123 = vcvtnq_s32_f32(vproduct4x0123);
vacc4x4567 = vcvtnq_s32_f32(vproduct4x4567);
vacc5x0123 = vcvtnq_s32_f32(vproduct5x0123);
vacc5x4567 = vcvtnq_s32_f32(vproduct5x4567);
const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->fp32_neonv8.output_zero_point);
#if XNN_ARCH_ARM64
const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point);
const int16x8_t vacc2x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc2x0123), vacc2x4567), voutput_zero_point);
const int16x8_t vacc3x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc3x0123), vacc3x4567), voutput_zero_point);
const int16x8_t vacc4x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc4x0123), vacc4x4567), voutput_zero_point);
const int16x8_t vacc5x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc5x0123), vacc5x4567), voutput_zero_point);
int8x16_t vout0x01234567_1x01234567 = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc1x01234567);
int8x16_t vout2x01234567_3x01234567 = vqmovn_high_s16(vqmovn_s16(vacc2x01234567), vacc3x01234567);
int8x16_t vout4x01234567_5x01234567 = vqmovn_high_s16(vqmovn_s16(vacc4x01234567), vacc5x01234567);
#else
const int16x8_t vacc0x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567)), voutput_zero_point);
const int16x8_t vacc1x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567)), voutput_zero_point);
const int16x8_t vacc2x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc2x0123), vqmovn_s32(vacc2x4567)), voutput_zero_point);
const int16x8_t vacc3x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc3x0123), vqmovn_s32(vacc3x4567)), voutput_zero_point);
const int16x8_t vacc4x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc4x0123), vqmovn_s32(vacc4x4567)), voutput_zero_point);
const int16x8_t vacc5x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc5x0123), vqmovn_s32(vacc5x4567)), voutput_zero_point);
int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567));
int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567));
int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567));
#endif
const int8x16_t voutput_min = vld1q_dup_s8(¶ms->fp32_neonv8.output_min);
const int8x16_t voutput_max = vld1q_dup_s8(¶ms->fp32_neonv8.output_max);
vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min);
vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min);
vout4x01234567_5x01234567 = vmaxq_s8(vout4x01234567_5x01234567, voutput_min);
vout0x01234567_1x01234567 = vminq_s8(vout0x01234567_1x01234567, voutput_max);
vout2x01234567_3x01234567 = vminq_s8(vout2x01234567_3x01234567, voutput_max);
vout4x01234567_5x01234567 = vminq_s8(vout4x01234567_5x01234567, voutput_max);
if (nc >= 8) {
// Main case where there the 8 columns fit in the destination.
vst1_s8(c0 + 0, vget_low_s8(vout0x01234567_1x01234567));
vst1_s8(c1 + 0, vget_high_s8(vout0x01234567_1x01234567));
vst1_s8(c2 + 0, vget_low_s8(vout2x01234567_3x01234567));
vst1_s8(c3 + 0, vget_high_s8(vout2x01234567_3x01234567));
vst1_s8(c4 + 0, vget_low_s8(vout4x01234567_5x01234567));
vst1_s8(c5 + 0, vget_high_s8(vout4x01234567_5x01234567));
// Advance to the next 8 columns.
c0 = (int8_t*) ((uintptr_t) c0 + cn_stride);
c1 = (int8_t*) ((uintptr_t) c1 + cn_stride);
c2 = (int8_t*) ((uintptr_t) c2 + cn_stride);
c3 = (int8_t*) ((uintptr_t) c3 + cn_stride);
c4 = (int8_t*) ((uintptr_t) c4 + cn_stride);
c5 = (int8_t*) ((uintptr_t) c5 + cn_stride);
a0 = (const int8_t*) ((uintptr_t) a0 - kc);
a1 = (const int8_t*) ((uintptr_t) a1 - kc);
a2 = (const int8_t*) ((uintptr_t) a2 - kc);
a3 = (const int8_t*) ((uintptr_t) a3 - kc);
a4 = (const int8_t*) ((uintptr_t) a4 - kc);
a5 = (const int8_t*) ((uintptr_t) a5 - kc);
nc -= 8;
} else {
// Final case where not all of the 8 columns fit in the destination.
if (nc & 4) {
vst1q_lane_u32(__builtin_assume_aligned(c0, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4;
vst1q_lane_u32(__builtin_assume_aligned(c1, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4;
vst1q_lane_u32(__builtin_assume_aligned(c2, 1), vreinterpretq_u32_s8(vout2x01234567_3x01234567), 0); c2 += 4;
vst1q_lane_u32(__builtin_assume_aligned(c3, 1), vreinterpretq_u32_s8(vout2x01234567_3x01234567), 2); c3 += 4;
vst1q_lane_u32(__builtin_assume_aligned(c4, 1), vreinterpretq_u32_s8(vout4x01234567_5x01234567), 0); c4 += 4;
vst1q_lane_u32(__builtin_assume_aligned(c5, 1), vreinterpretq_u32_s8(vout4x01234567_5x01234567), 2); c5 += 4;
vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4);
vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 4);
vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 4);
}
if (nc & 2) {
vst1q_lane_u16(__builtin_assume_aligned(c0, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2;
vst1q_lane_u16(__builtin_assume_aligned(c1, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2;
vst1q_lane_u16(__builtin_assume_aligned(c2, 1), vreinterpretq_u16_s8(vout2x01234567_3x01234567), 0); c2 += 2;
vst1q_lane_u16(__builtin_assume_aligned(c3, 1), vreinterpretq_u16_s8(vout2x01234567_3x01234567), 4); c3 += 2;
vst1q_lane_u16(__builtin_assume_aligned(c4, 1), vreinterpretq_u16_s8(vout4x01234567_5x01234567), 0); c4 += 2;
vst1q_lane_u16(__builtin_assume_aligned(c5, 1), vreinterpretq_u16_s8(vout4x01234567_5x01234567), 4); c5 += 2;
vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2);
vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 2);
vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 2);
}
if (nc & 1) {
vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0);
vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8);
vst1q_lane_s8(c2, vout2x01234567_3x01234567, 0);
vst1q_lane_s8(c3, vout2x01234567_3x01234567, 8);
vst1q_lane_s8(c4, vout4x01234567_5x01234567, 0);
vst1q_lane_s8(c5, vout4x01234567_5x01234567, 8);
}
nc = 0;
}
} while (nc != 0);
}