.global RestoreRegisters_NEON
.global ReverseLine_NEON
.global ReverseLineUV_NEON
.global SaveRegisters_NEON
.global TransposeWx8_NEON
.global TransposeUVWx8_NEON
.type RestoreRegisters_NEON, function
.type ReverseLine_NEON, function
.type ReverseLineUV_NEON, function
.type SaveRegisters_NEON, function
.type TransposeWx8_NEON, function
.type TransposeUVWx8_NEON, function
@ void ReverseLine_NEON (const uint8* src, uint8* dst, int width)
@ r0 const uint8* src
@ r1 uint8* dst
@ r2 width
ReverseLine_NEON:
@ compute where to start writing destination
add r1, r2 @ dst + width
@ work on segments that are multiples of 16
lsrs r3, r2, #4
@ the output is written in two block. 8 bytes followed
@ by another 8. reading is done sequentially, from left to
@ right. writing is done from right to left in block sizes
@ r1, the destination pointer is incremented after writing
@ the first of the two blocks. need to subtract that 8 off
@ along with 16 to get the next location.
mov r3, #-24
beq Lline_residuals
@ back of destination by the size of the register that is
@ going to be reversed
sub r1, #16
@ the loop needs to run on blocks of 16. what will be left
@ over is either a negative number, the residuals that need
@ to be done, or 0. if this isn't subtracted off here the
@ loop will run one extra time.
sub r2, #16
Lsegments_of_16:
vld1.8 {q0}, [r0]! @ src += 16
@ reverse the bytes in the 64 bit segments. unable to reverse
@ the bytes in the entire 128 bits in one go.
vrev64.8 q0, q0
@ because of the inability to reverse the entire 128 bits
@ reverse the writing out of the two 64 bit segments.
vst1.8 {d1}, [r1]!
vst1.8 {d0}, [r1], r3 @ dst -= 16
subs r2, #16
bge Lsegments_of_16
@ add 16 back to the counter. if the result is 0 there is no
@ residuals so return
adds r2, #16
bxeq lr
add r1, #16
Lline_residuals:
mov r3, #-3
sub r1, #2
subs r2, #2
@ check for 16*n+1 scenarios where segments_of_2 should not
@ be run, but there is something left over.
blt Lsegment_of_1
@ do this in neon registers as per
@ http://blogs.arm.com/software-enablement/196-coding-for-neon-part-2-dealing-with-leftovers/
Lsegments_of_2:
vld2.8 {d0[0], d1[0]}, [r0]! @ src += 2
vst1.8 {d1[0]}, [r1]!
vst1.8 {d0[0]}, [r1], r3 @ dst -= 2
subs r2, #2
bge Lsegments_of_2
adds r2, #2
bxeq lr
Lsegment_of_1:
add r1, #1
vld1.8 {d0[0]}, [r0]
vst1.8 {d0[0]}, [r1]
bx lr
@ void TransposeWx8_NEON (const uint8* src, int src_stride,
@ uint8* dst, int dst_stride,
@ int w)
@ r0 const uint8* src
@ r1 int src_stride
@ r2 uint8* dst
@ r3 int dst_stride
@ stack int w
TransposeWx8_NEON:
push {r4,r8,r9,lr}
ldr r8, [sp, #16] @ width
@ loops are on blocks of 8. loop will stop when
@ counter gets to or below 0. starting the counter
@ at w-8 allow for this
sub r8, #8
@ handle 8x8 blocks. this should be the majority of the plane
Lloop_8x8:
mov r9, r0
vld1.8 {d0}, [r9], r1
vld1.8 {d1}, [r9], r1
vld1.8 {d2}, [r9], r1
vld1.8 {d3}, [r9], r1
vld1.8 {d4}, [r9], r1
vld1.8 {d5}, [r9], r1
vld1.8 {d6}, [r9], r1
vld1.8 {d7}, [r9]
vtrn.8 d1, d0
vtrn.8 d3, d2
vtrn.8 d5, d4
vtrn.8 d7, d6
vtrn.16 d1, d3
vtrn.16 d0, d2
vtrn.16 d5, d7
vtrn.16 d4, d6
vtrn.32 d1, d5
vtrn.32 d0, d4
vtrn.32 d3, d7
vtrn.32 d2, d6
vrev16.8 q0, q0
vrev16.8 q1, q1
vrev16.8 q2, q2
vrev16.8 q3, q3
mov r9, r2
vst1.8 {d1}, [r9], r3
vst1.8 {d0}, [r9], r3
vst1.8 {d3}, [r9], r3
vst1.8 {d2}, [r9], r3
vst1.8 {d5}, [r9], r3
vst1.8 {d4}, [r9], r3
vst1.8 {d7}, [r9], r3
vst1.8 {d6}, [r9]
add r0, #8 @ src += 8
add r2, r3, lsl #3 @ dst += 8 * dst_stride
subs r8, #8 @ w -= 8
bge Lloop_8x8
@ add 8 back to counter. if the result is 0 there are
@ no residuals.
adds r8, #8
beq Ldone
@ some residual, so between 1 and 7 lines left to transpose
cmp r8, #2
blt Lblock_1x8
cmp r8, #4
blt Lblock_2x8
Lblock_4x8:
mov r9, r0
vld1.32 {d0[0]}, [r9], r1
vld1.32 {d0[1]}, [r9], r1
vld1.32 {d1[0]}, [r9], r1
vld1.32 {d1[1]}, [r9], r1
vld1.32 {d2[0]}, [r9], r1
vld1.32 {d2[1]}, [r9], r1
vld1.32 {d3[0]}, [r9], r1
vld1.32 {d3[1]}, [r9]
mov r9, r2
adr r12, vtbl_4x4_transpose
vld1.8 {q3}, [r12]
vtbl.8 d4, {d0, d1}, d6
vtbl.8 d5, {d0, d1}, d7
vtbl.8 d0, {d2, d3}, d6
vtbl.8 d1, {d2, d3}, d7
@ TODO: rework shuffle above to write
@ out with 4 instead of 8 writes
vst1.32 {d4[0]}, [r9], r3
vst1.32 {d4[1]}, [r9], r3
vst1.32 {d5[0]}, [r9], r3
vst1.32 {d5[1]}, [r9]
add r9, r2, #4
vst1.32 {d0[0]}, [r9], r3
vst1.32 {d0[1]}, [r9], r3
vst1.32 {d1[0]}, [r9], r3
vst1.32 {d1[1]}, [r9]
add r0, #4 @ src += 4
add r2, r3, lsl #2 @ dst += 4 * dst_stride
subs r8, #4 @ w -= 4
beq Ldone
@ some residual, check to see if it includes a 2x8 block,
@ or less
cmp r8, #2
blt Lblock_1x8
Lblock_2x8:
mov r9, r0
vld1.16 {d0[0]}, [r9], r1
vld1.16 {d1[0]}, [r9], r1
vld1.16 {d0[1]}, [r9], r1
vld1.16 {d1[1]}, [r9], r1
vld1.16 {d0[2]}, [r9], r1
vld1.16 {d1[2]}, [r9], r1
vld1.16 {d0[3]}, [r9], r1
vld1.16 {d1[3]}, [r9]
vtrn.8 d0, d1
mov r9, r2
vst1.64 {d0}, [r9], r3
vst1.64 {d1}, [r9]
add r0, #2 @ src += 2
add r2, r3, lsl #1 @ dst += 2 * dst_stride
subs r8, #2 @ w -= 2
beq Ldone
Lblock_1x8:
vld1.8 {d0[0]}, [r0], r1
vld1.8 {d0[1]}, [r0], r1
vld1.8 {d0[2]}, [r0], r1
vld1.8 {d0[3]}, [r0], r1
vld1.8 {d0[4]}, [r0], r1
vld1.8 {d0[5]}, [r0], r1
vld1.8 {d0[6]}, [r0], r1
vld1.8 {d0[7]}, [r0]
vst1.64 {d0}, [r2]
Ldone:
pop {r4,r8,r9,pc}
vtbl_4x4_transpose:
.byte 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15
@ void SaveRegisters_NEON (unsigned long long store)
@ r0 unsigned long long store
SaveRegisters_NEON:
vst1.i64 {d8, d9, d10, d11}, [r0]!
vst1.i64 {d12, d13, d14, d15}, [r0]!
bx lr
@ void RestoreRegisters_NEON (unsigned long long store)
@ r0 unsigned long long store
RestoreRegisters_NEON:
vld1.i64 {d8, d9, d10, d11}, [r0]!
vld1.i64 {d12, d13, d14, d15}, [r0]!
bx lr
@ void ReverseLineUV_NEON (const uint8* src,
@ uint8* dst_a,
@ uint8* dst_b,
@ int width)
@ r0 const uint8* src
@ r1 uint8* dst_a
@ r2 uint8* dst_b
@ r3 width
ReverseLineUV_NEON:
@ compute where to start writing destination
add r1, r1, r3 @ dst_a + width
add r2, r2, r3 @ dst_b + width
@ work on input segments that are multiples of 16, but
@ width that has been passed is output segments, half
@ the size of input.
lsrs r12, r3, #3
beq Lline_residuals_di
@ the output is written in to two blocks.
mov r12, #-8
@ back of destination by the size of the register that is
@ going to be reversed
sub r1, r1, #8
sub r2, r2, #8
@ the loop needs to run on blocks of 8. what will be left
@ over is either a negative number, the residuals that need
@ to be done, or 0. if this isn't subtracted off here the
@ loop will run one extra time.
sub r3, r3, #8
Lsegments_of_8_di:
vld2.8 {d0, d1}, [r0]! @ src += 16
@ reverse the bytes in the 64 bit segments
vrev64.8 q0, q0
vst1.8 {d0}, [r1], r12 @ dst_a -= 8
vst1.8 {d1}, [r2], r12 @ dst_b -= 8
subs r3, r3, #8
bge Lsegments_of_8_di
@ add 8 back to the counter. if the result is 0 there is no
@ residuals so return
adds r3, r3, #8
bxeq lr
add r1, r1, #8
add r2, r2, #8
Lline_residuals_di:
mov r12, #-1
sub r1, r1, #1
sub r2, r2, #1
@ do this in neon registers as per
@ http://blogs.arm.com/software-enablement/196-coding-for-neon-part-2-dealing-with-leftovers/
Lsegments_of_1:
vld2.8 {d0[0], d1[0]}, [r0]! @ src += 2
vst1.8 {d0[0]}, [r1], r12 @ dst_a -= 1
vst1.8 {d1[0]}, [r2], r12 @ dst_b -= 1
subs r3, r3, #1
bgt Lsegments_of_1
bx lr
@ void TransposeUVWx8_NEON (const uint8* src, int src_stride,
@ uint8* dst_a, int dst_stride_a,
@ uint8* dst_b, int dst_stride_b,
@ int width)
@ r0 const uint8* src
@ r1 int src_stride
@ r2 uint8* dst_a
@ r3 int dst_stride_a
@ stack uint8* dst_b
@ stack int dst_stride_b
@ stack int width
TransposeUVWx8_NEON:
push {r4-r9,lr}
ldr r4, [sp, #28] @ dst_b
ldr r5, [sp, #32] @ dst_stride_b
ldr r8, [sp, #36] @ width
@ loops are on blocks of 8. loop will stop when
@ counter gets to or below 0. starting the counter
@ at w-8 allow for this
sub r8, #8
@ handle 8x8 blocks. this should be the majority of the plane
Lloop_8x8_di:
mov r9, r0
vld2.8 {d0, d1}, [r9], r1
vld2.8 {d2, d3}, [r9], r1
vld2.8 {d4, d5}, [r9], r1
vld2.8 {d6, d7}, [r9], r1
vld2.8 {d8, d9}, [r9], r1
vld2.8 {d10, d11}, [r9], r1
vld2.8 {d12, d13}, [r9], r1
vld2.8 {d14, d15}, [r9]
vtrn.8 q1, q0
vtrn.8 q3, q2
vtrn.8 q5, q4
vtrn.8 q7, q6
vtrn.16 q1, q3
vtrn.16 q0, q2
vtrn.16 q5, q7
vtrn.16 q4, q6
vtrn.32 q1, q5
vtrn.32 q0, q4
vtrn.32 q3, q7
vtrn.32 q2, q6
vrev16.8 q0, q0
vrev16.8 q1, q1
vrev16.8 q2, q2
vrev16.8 q3, q3
vrev16.8 q4, q4
vrev16.8 q5, q5
vrev16.8 q6, q6
vrev16.8 q7, q7
mov r9, r2
vst1.8 {d2}, [r9], r3
vst1.8 {d0}, [r9], r3
vst1.8 {d6}, [r9], r3
vst1.8 {d4}, [r9], r3
vst1.8 {d10}, [r9], r3
vst1.8 {d8}, [r9], r3
vst1.8 {d14}, [r9], r3
vst1.8 {d12}, [r9]
mov r9, r4
vst1.8 {d3}, [r9], r5
vst1.8 {d1}, [r9], r5
vst1.8 {d7}, [r9], r5
vst1.8 {d5}, [r9], r5
vst1.8 {d11}, [r9], r5
vst1.8 {d9}, [r9], r5
vst1.8 {d15}, [r9], r5
vst1.8 {d13}, [r9]
add r0, #8*2 @ src += 8*2
add r2, r3, lsl #3 @ dst_a += 8 * dst_stride_a
add r4, r5, lsl #3 @ dst_b += 8 * dst_stride_b
subs r8, #8 @ w -= 8
bge Lloop_8x8_di
@ add 8 back to counter. if the result is 0 there are
@ no residuals.
adds r8, #8
beq Ldone_di
@ some residual, so between 1 and 7 lines left to transpose
cmp r8, #2
blt Lblock_1x8_di
cmp r8, #4
blt Lblock_2x8_di
@ TODO(frkoenig) : clean this up
Lblock_4x8_di:
mov r9, r0
vld1.64 {d0}, [r9], r1
vld1.64 {d1}, [r9], r1
vld1.64 {d2}, [r9], r1
vld1.64 {d3}, [r9], r1
vld1.64 {d4}, [r9], r1
vld1.64 {d5}, [r9], r1
vld1.64 {d6}, [r9], r1
vld1.64 {d7}, [r9]
adr r12, vtbl_4x4_transpose_di
vld1.8 {q7}, [r12]
vtrn.8 q0, q1
vtrn.8 q2, q3
vtbl.8 d8, {d0, d1}, d14
vtbl.8 d9, {d0, d1}, d15
vtbl.8 d10, {d2, d3}, d14
vtbl.8 d11, {d2, d3}, d15
vtbl.8 d12, {d4, d5}, d14
vtbl.8 d13, {d4, d5}, d15
vtbl.8 d0, {d6, d7}, d14
vtbl.8 d1, {d6, d7}, d15
mov r9, r2
vst1.32 {d8[0]}, [r9], r3
vst1.32 {d8[1]}, [r9], r3
vst1.32 {d9[0]}, [r9], r3
vst1.32 {d9[1]}, [r9], r3
add r9, r2, #4
vst1.32 {d12[0]}, [r9], r3
vst1.32 {d12[1]}, [r9], r3
vst1.32 {d13[0]}, [r9], r3
vst1.32 {d13[1]}, [r9]
mov r9, r4
vst1.32 {d10[0]}, [r9], r5
vst1.32 {d10[1]}, [r9], r5
vst1.32 {d11[0]}, [r9], r5
vst1.32 {d11[1]}, [r9], r5
add r9, r4, #4
vst1.32 {d0[0]}, [r9], r5
vst1.32 {d0[1]}, [r9], r5
vst1.32 {d1[0]}, [r9], r5
vst1.32 {d1[1]}, [r9]
add r0, #4*2 @ src += 4 * 2
add r2, r3, lsl #2 @ dst_a += 4 * dst_stride_a
add r4, r5, lsl #2 @ dst_b += 4 * dst_stride_b
subs r8, #4 @ w -= 4
beq Ldone_di
@ some residual, check to see if it includes a 2x8 block,
@ or less
cmp r8, #2
blt Lblock_1x8_di
Lblock_2x8_di:
mov r9, r0
vld2.16 {d0[0], d2[0]}, [r9], r1
vld2.16 {d1[0], d3[0]}, [r9], r1
vld2.16 {d0[1], d2[1]}, [r9], r1
vld2.16 {d1[1], d3[1]}, [r9], r1
vld2.16 {d0[2], d2[2]}, [r9], r1
vld2.16 {d1[2], d3[2]}, [r9], r1
vld2.16 {d0[3], d2[3]}, [r9], r1
vld2.16 {d1[3], d3[3]}, [r9]
vtrn.8 d0, d1
vtrn.8 d2, d3
mov r9, r2
vst1.64 {d0}, [r9], r3
vst1.64 {d2}, [r9]
mov r9, r4
vst1.64 {d1}, [r9], r5
vst1.64 {d3}, [r9]
add r0, #2*2 @ src += 2 * 2
add r2, r3, lsl #1 @ dst_a += 2 * dst_stride_a
add r4, r5, lsl #1 @ dst_a += 2 * dst_stride_a
subs r8, #2 @ w -= 2
beq Ldone_di
Lblock_1x8_di:
vld2.8 {d0[0], d1[0]}, [r0], r1
vld2.8 {d0[1], d1[1]}, [r0], r1
vld2.8 {d0[2], d1[2]}, [r0], r1
vld2.8 {d0[3], d1[3]}, [r0], r1
vld2.8 {d0[4], d1[4]}, [r0], r1
vld2.8 {d0[5], d1[5]}, [r0], r1
vld2.8 {d0[6], d1[6]}, [r0], r1
vld2.8 {d0[7], d1[7]}, [r0]
vst1.64 {d0}, [r2]
vst1.64 {d1}, [r4]
Ldone_di:
pop {r4-r9, pc}
vtbl_4x4_transpose_di:
.byte 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15