From 9f95ff5b6ba01db09552b84a0ab79607060a2666 Mon Sep 17 00:00:00 2001 From: Ali Labbene Date: Wed, 11 Dec 2019 08:59:21 +0100 Subject: Official ARM version: v5.4.0 Add CMSIS V5.4.0, please refer to index.html available under \docs folder. Note: content of \CMSIS\Core\Include has been copied under \Include to keep the same structure used in existing projects, and thus avoid projects mass update Note: the following components have been removed from ARM original delivery (as not used in ST packages) - CMSIS_EW2018.pdf - .gitattributes - .gitignore - \Device - \CMSIS - \CoreValidation - \DAP - \Documentation - \DoxyGen - \Driver - \Pack - \RTOS\CMSIS_RTOS_Tutorial.pdf - \RTOS\RTX - \RTOS\Template - \RTOS2\RTX - \Utilities - All ARM/GCC projects files are deleted from \DSP, \RTOS and \RTOS2 Change-Id: Ia026c3f0f0d016627a4fb5a9032852c33d24b4d3 --- .../ActivationFunctions/arm_nn_activations_q15.c | 101 +++++ .../ActivationFunctions/arm_nn_activations_q7.c | 91 ++++ NN/Source/ActivationFunctions/arm_relu_q15.c | 106 +++++ NN/Source/ActivationFunctions/arm_relu_q7.c | 110 +++++ .../arm_convolve_1x1_HWC_q7_fast_nonsquare.c | 235 ++++++++++ .../arm_convolve_HWC_q15_basic.c | 207 +++++++++ .../arm_convolve_HWC_q15_fast.c | 255 +++++++++++ .../arm_convolve_HWC_q15_fast_nonsquare.c | 265 +++++++++++ .../ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c | 279 ++++++++++++ .../arm_convolve_HWC_q7_basic.c | 230 ++++++++++ .../arm_convolve_HWC_q7_basic_nonsquare.c | 228 ++++++++++ .../arm_convolve_HWC_q7_fast.c | 408 +++++++++++++++++ .../arm_convolve_HWC_q7_fast_nonsquare.c | 379 ++++++++++++++++ .../arm_depthwise_separable_conv_HWC_q7.c | 418 ++++++++++++++++++ ...arm_depthwise_separable_conv_HWC_q7_nonsquare.c | 411 +++++++++++++++++ .../arm_nn_mat_mult_kernel_q7_q15.c | 187 ++++++++ .../arm_nn_mat_mult_kernel_q7_q15_reordered.c | 138 ++++++ .../arm_fully_connected_mat_q7_vec_q15.c | 199 +++++++++ .../arm_fully_connected_mat_q7_vec_q15_opt.c | 403 +++++++++++++++++ .../arm_fully_connected_q15.c | 193 ++++++++ .../arm_fully_connected_q15_opt.c | 332 ++++++++++++++ .../arm_fully_connected_q7.c | 198 +++++++++ .../arm_fully_connected_q7_opt.c | 484 +++++++++++++++++++++ NN/Source/NNSupportFunctions/arm_nn_mult_q15.c | 147 +++++++ NN/Source/NNSupportFunctions/arm_nn_mult_q7.c | 119 +++++ NN/Source/NNSupportFunctions/arm_nntables.c | 297 +++++++++++++ .../NNSupportFunctions/arm_q7_to_q15_no_shift.c | 134 ++++++ .../arm_q7_to_q15_reordered_no_shift.c | 145 ++++++ NN/Source/PoolingFunctions/arm_pool_q7_HWC.c | 448 +++++++++++++++++++ NN/Source/SoftmaxFunctions/arm_softmax_q15.c | 120 +++++ NN/Source/SoftmaxFunctions/arm_softmax_q7.c | 121 ++++++ 31 files changed, 7388 insertions(+) create mode 100644 NN/Source/ActivationFunctions/arm_nn_activations_q15.c create mode 100644 NN/Source/ActivationFunctions/arm_nn_activations_q7.c create mode 100644 NN/Source/ActivationFunctions/arm_relu_q15.c create mode 100644 NN/Source/ActivationFunctions/arm_relu_q7.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c create mode 100644 NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c create mode 100644 NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c create mode 100644 NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c create mode 100644 NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c create mode 100644 NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c create mode 100644 NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c create mode 100644 NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c create mode 100644 NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c create mode 100644 NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c create mode 100644 NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c create mode 100644 NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c create mode 100644 NN/Source/NNSupportFunctions/arm_nn_mult_q15.c create mode 100644 NN/Source/NNSupportFunctions/arm_nn_mult_q7.c create mode 100644 NN/Source/NNSupportFunctions/arm_nntables.c create mode 100644 NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c create mode 100644 NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c create mode 100644 NN/Source/PoolingFunctions/arm_pool_q7_HWC.c create mode 100644 NN/Source/SoftmaxFunctions/arm_softmax_q15.c create mode 100644 NN/Source/SoftmaxFunctions/arm_softmax_q7.c (limited to 'NN/Source') diff --git a/NN/Source/ActivationFunctions/arm_nn_activations_q15.c b/NN/Source/ActivationFunctions/arm_nn_activations_q15.c new file mode 100644 index 0000000..9c64e2a --- /dev/null +++ b/NN/Source/ActivationFunctions/arm_nn_activations_q15.c @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_nn_activations_q15.c + * Description: Q15 neural network activation function using direct table look-up + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_common_tables.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup Acti + * @{ + */ + + /** + * @brief Q15 neural network activation function using direct table look-up + * @param[in,out] data pointer to input + * @param[in] size number of elements + * @param[in] int_width bit-width of the integer part, assume to be smaller than 3 + * @param[in] type type of activation functions + * @return none. + * + * @details + * + * This is the direct table look-up approach. + * + * Assume here the integer part of the fixed-point is <= 3. + * More than 3 just not making much sense, makes no difference with + * saturation followed by any of these activation functions. + */ + +void arm_nn_activations_direct_q15(q15_t * data, uint16_t size, uint16_t int_width, arm_nn_activation_type type) +{ + uint16_t i = size; + q15_t *pIn = data; + q15_t *pOut = data; + uint16_t shift_size = 8 + 3 - int_width; + uint32_t bit_mask = 0x7FF >> int_width; + uint32_t full_frac = bit_mask + 1; + const q15_t *lookup_table; + + switch (type) + { + case ARM_SIGMOID: + lookup_table = sigmoidTable_q15; + break; + case ARM_TANH: + default: + lookup_table = tanhTable_q15; + break; + } + + while (i) + { + q15_t out; + q15_t in = *pIn++; + q15_t frac = (uint32_t) in & bit_mask; + q15_t value = lookup_table[__USAT(in >> shift_size, 8)]; + q15_t value2 = lookup_table[__USAT(1 + (in >> shift_size), 8)]; + + /* doing the interpolation here for better accuracy */ + out = ((q31_t) (full_frac - frac) * value + (q31_t) value2 * frac) >> shift_size; + + *pOut++ = out; + i--; + } + +} + +/** + * @} end of Acti group + */ diff --git a/NN/Source/ActivationFunctions/arm_nn_activations_q7.c b/NN/Source/ActivationFunctions/arm_nn_activations_q7.c new file mode 100644 index 0000000..1ca429f --- /dev/null +++ b/NN/Source/ActivationFunctions/arm_nn_activations_q7.c @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_nn_activations_q7.c + * Description: Q7 neural network activation function using direct table look-up + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_common_tables.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup Acti + * @{ + */ + + /** + * @brief Q7 neural network activation function using direct table look-up + * @param[in,out] data pointer to input + * @param[in] size number of elements + * @param[in] int_width bit-width of the integer part, assume to be smaller than 3 + * @param[in] type type of activation functions + * @return none. + * + * @details + * + * This is the direct table look-up approach. + * + * Assume here the integer part of the fixed-point is <= 3. + * More than 3 just not making much sense, makes no difference with + * saturation followed by any of these activation functions. + */ + +void arm_nn_activations_direct_q7(q7_t * data, uint16_t size, uint16_t int_width, arm_nn_activation_type type) +{ + uint16_t i = size; + q7_t *pIn = data; + q7_t *pOut = data; + q7_t in; + q7_t out; + uint16_t shift_size = 3 - int_width; + const q7_t *lookup_table; + switch (type) + { + case ARM_SIGMOID: + lookup_table = sigmoidTable_q7; + break; + case ARM_TANH: + default: + lookup_table = tanhTable_q7; + break; + } + while (i) + { + in = *pIn++; + out = lookup_table[(uint8_t) (in >> shift_size)]; + *pOut++ = out; + i--; + } +} + +/** + * @} end of Acti group + */ diff --git a/NN/Source/ActivationFunctions/arm_relu_q15.c b/NN/Source/ActivationFunctions/arm_relu_q15.c new file mode 100644 index 0000000..571d51c --- /dev/null +++ b/NN/Source/ActivationFunctions/arm_relu_q15.c @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_relu_q15.c + * Description: Q15 version of ReLU + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup Acti + * @{ + */ + + /** + * @brief Q15 RELU function + * @param[in,out] data pointer to input + * @param[in] size number of elements + * @return none. + * + * @details + * + * Optimized relu with QSUB instructions. + * + */ + +void arm_relu_q15(q15_t * data, uint16_t size) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + uint16_t i = size >> 1; + q15_t *pIn = data; + q15_t *pOut = data; + q31_t in; + q31_t buf; + q31_t mask; + + while (i) + { + in = *__SIMD32(pIn)++; + + /* extract the first bit */ + buf = __ROR(in & 0x80008000, 15); + + /* if MSB=1, mask will be 0xFF, 0x0 otherwise */ + mask = __QSUB16(0x00000000, buf); + + *__SIMD32(pOut)++ = in & (~mask); + i--; + } + + if (size & 0x1) + { + if (*pIn < 0) + { + *pIn = 0; + } + pIn++; + } +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + uint16_t i; + + for (i = 0; i < size; i++) + { + if (data[i] < 0) + data[i] = 0; + } + +#endif /* ARM_MATH_DSP */ + +} + +/** + * @} end of Acti group + */ diff --git a/NN/Source/ActivationFunctions/arm_relu_q7.c b/NN/Source/ActivationFunctions/arm_relu_q7.c new file mode 100644 index 0000000..013325c --- /dev/null +++ b/NN/Source/ActivationFunctions/arm_relu_q7.c @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_relu_q7.c + * Description: Q7 version of ReLU + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup Acti + * @{ + */ + + /** + * @brief Q7 RELU function + * @param[in,out] data pointer to input + * @param[in] size number of elements + * @return none. + * + * @details + * + * Optimized relu with QSUB instructions. + * + */ + +void arm_relu_q7(q7_t * data, uint16_t size) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + uint16_t i = size >> 2; + q7_t *pIn = data; + q7_t *pOut = data; + q31_t in; + q31_t buf; + q31_t mask; + + while (i) + { + in = *__SIMD32(pIn)++; + + /* extract the first bit */ + buf = __ROR(in & 0x80808080, 7); + + /* if MSB=1, mask will be 0xFF, 0x0 otherwise */ + mask = __QSUB8(0x00000000, buf); + + *__SIMD32(pOut)++ = in & (~mask); + i--; + } + + i = size & 0x3; + while (i) + { + if (*pIn < 0) + { + *pIn = 0; + } + pIn++; + i--; + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + uint16_t i; + + for (i = 0; i < size; i++) + { + if (data[i] < 0) + data[i] = 0; + } + +#endif /* ARM_MATH_DSP */ + +} + +/** + * @} end of Acti group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c b/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c new file mode 100644 index 0000000..2f4133c --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c @@ -0,0 +1,235 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_1x1_HWC_q7_fast_nonsquare.c + * Description: Fast Q7 version of 1x1 convolution (non-square shape) + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + +/** + * @brief Fast Q7 version of 1x1 convolution (non-sqaure shape) + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in_x input tensor dimention x + * @param[in] dim_im_in_y input tensor dimention y + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel_x filter kernel size x + * @param[in] dim_kernel_y filter kernel size y + * @param[in] padding_x padding size x + * @param[in] padding_y padding size y + * @param[in] stride_x convolution stride x + * @param[in] stride_y convolution stride y + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out_x output tensor dimension x + * @param[in] dim_im_out_y output tensor dimension y + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * This function is optimized for convolution with 1x1 kernel size (i.e., dim_kernel_x=1 + * and dim_kernel_y=1). It can be used for the second half of MobileNets [1] after depthwise + * separable convolution. + * + * This function is the version with full list of optimization tricks, but with + * some contraints: + * ch_im_in is multiple of 4 + * ch_im_out is multiple of 2 + * + * [1] MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications + * https://arxiv.org/abs/1704.04861 + */ + +arm_status arm_convolve_1x1_HWC_q7_fast_nonsquare(const q7_t * Im_in, + const uint16_t dim_im_in_x, + const uint16_t dim_im_in_y, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel_x, + const uint16_t dim_kernel_y, + const uint16_t padding_x, + const uint16_t padding_y, + const uint16_t stride_x, + const uint16_t stride_y, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, + const uint16_t dim_im_out_x, + const uint16_t dim_im_out_y, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_out_y, i_out_x; + int16_t i_ch_out; + + /* ----------------------- + * Here we use bufferA as q15_t internally as computation are done with q15_t level + * im2col are done to output in q15_t format from q7_t input + */ + + q15_t *pBuffer = bufferA; + q7_t *pOut = Im_out; + + if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1 + || padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) + { + /* This part implements the im2col function */ + arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_out_y * dim_im_in_x + i_out_x) * ch_im_in, pBuffer, + ch_im_in); + pBuffer += ch_im_in; + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in, bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* check if there is left-over for compute */ + if (pBuffer != bufferA) + { + const q7_t *pA = wt; + for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++) + { + q31_t sum = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift); + q15_t *pB = bufferA; + /* basically each time it process 4 entries */ + uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2; + + while (colCnt) + { + + q31_t inA1, inA2; + q31_t inB1, inB2; + + pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2); + + inB1 = *__SIMD32(pB)++; + sum = __SMLAD(inA1, inB1, sum); + inB2 = *__SIMD32(pB)++; + sum = __SMLAD(inA2, inB2, sum); + + colCnt--; + } + colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + sum += inA1 * inB1; + colCnt--; + } + *pOut = (q7_t) __SSAT((sum >> out_shift), 8); + pOut++; + + } + + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + int i, j, k, l, m, n; + int conv_out; + int in_row, in_col; + + if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1 + || padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out_y; j++) + { + for (k = 0; k < dim_im_out_x; k++) + { + conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel_y; m++) + { + for (n = 0; n < dim_kernel_x; n++) + { + // if-for implementation + in_row = stride_y * j + m - padding_y; + in_col = stride_x * k + n - padding_x; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] * + wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_y + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c new file mode 100644 index 0000000..00b5aa5 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q15_basic.c + * Description: Q15 version of convolution + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + + /** + * @brief Basic Q15 convolution function + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns ARM_MATH_SUCCESS + * + * @details + * + * Buffer size: + * + * bufferA size: ch_im_in*dim_kernel*dim_kernel + * + * bufferB size: 0 + * + * This basic version is designed to work for any input tensor and weight + * dimension. + */ + +arm_status +arm_convolve_HWC_q15_basic(const q15_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const q15_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, + const q15_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q15_t * Im_out, + const uint16_t dim_im_out, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + uint16_t im2col_out_pixel_index = 0; + q15_t *pBuffer = bufferA; + q15_t *pOut = Im_out; + q15_t *im_buffer = bufferA; + const q15_t *pA; + int i; + + /* This part implements the im2col function */ + for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* Filling 0 for out-of-bound paddings */ + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */ + memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in); + } + pBuffer += ch_im_in; + } + } + + pA = wt; + for (i = 0; i < ch_im_out; i++) + { + q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + q15_t *pB = im_buffer; + uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2; + while (colCnt) + { + q31_t inA1 = *__SIMD32(pA)++; + q31_t inB1 = *__SIMD32(pB)++; + q31_t inA2 = *__SIMD32(pA)++; + q31_t inB2 = *__SIMD32(pB)++; + + sum = __SMLAD(inA1, inB1, sum); + sum = __SMLAD(inA2, inB2, sum); + + colCnt--; + } + colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3; + while (colCnt) + { + q15_t inA1 = *pA++; + q15_t inB1 = *pB++; + sum += inA1 * inB1; + colCnt--; + } + *pOut = (q15_t) __SSAT((sum >> out_shift), 16); + pOut++; + } + + /* counter reset */ + pBuffer = im_buffer; + im2col_out_pixel_index++; + } + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + uint16_t i, j, k, l, m, n; + int conv_out; + signed char in_row, in_col; + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out; j++) + { + for (k = 0; k < dim_im_out; k++) + { + conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel; m++) + { + for (n = 0; n < dim_kernel; n++) + { + in_row = stride * j + m - padding; + in_col = stride * k + n - padding; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += + Im_in[(in_row * dim_im_in + in_col) * ch_im_in + + l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c new file mode 100644 index 0000000..c9873c1 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q15_fast.c + * Description: Fast Q15 version of convolution + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + + /** + * @brief Fast Q15 convolution function + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * @details + * + * Buffer size: + * + * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel + * + * bufferB size: 0 + * + * Input dimension constraints: + * + * ch_im_in is multiple of 2 + * + * ch_im_out is multipe of 2 + * + */ + +arm_status +arm_convolve_HWC_q15_fast(const q15_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const q15_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, + const q15_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q15_t * Im_out, + const uint16_t dim_im_out, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + q15_t *pBuffer = bufferA; + q15_t *im_buffer = bufferA; + q15_t *pOut = Im_out; + + if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + /* This part implements the im2col function */ + for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */ + memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (i_out_x & 0x1) + { + int i; + /* initialize the matrix pointers for A */ + const q15_t *pA = wt; + + /* set up the second output pointers */ + q15_t *pOut2 = pOut + ch_im_out; + + /* this loop over rows in A */ + for (i = 0; i < ch_im_out; i += 2) + { + /* setup pointers for B */ + q15_t *pB = im_buffer; + const q15_t *pB2 = pB + ch_im_in * dim_kernel * dim_kernel; + + /* aling the second pointer for A */ + const q15_t *pA2 = pA + ch_im_in * dim_kernel * dim_kernel; + + /* init the sum with bias */ + q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 1; + /* accumulate over the vector */ + while (colCnt) + { + q31_t inA1 = *__SIMD32(pA)++; + q31_t inB1 = *__SIMD32(pB)++; + q31_t inA2 = *__SIMD32(pA2)++; + q31_t inB2 = *__SIMD32(pB2)++; + + sum = __SMLAD(inA1, inB1, sum); + sum2 = __SMLAD(inA1, inB2, sum2); + sum3 = __SMLAD(inA2, inB1, sum3); + sum4 = __SMLAD(inA2, inB2, sum4); + + colCnt--; + } /* while over colCnt */ + colCnt = ch_im_in * dim_kernel * dim_kernel & 0x1; + while (colCnt) + { + q15_t inA1 = *pA++; + q15_t inB1 = *pB++; + q15_t inA2 = *pA2++; + q15_t inB2 = *pB2++; + + sum += inA1 * inB1; + sum2 += inA1 * inB2; + sum3 += inA2 * inB1; + sum4 += inA2 * inB2; + colCnt--; + } /* while over colCnt */ + *pOut++ = (q15_t) __SSAT(sum >> out_shift, 16); + *pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16); + *pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16); + *pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16); + + /* skip the row computed with A2 */ + pA += ch_im_in * dim_kernel * dim_kernel; + } /* for over ch_im_out */ + + pOut += ch_im_out; + /* counter reset */ + pBuffer = im_buffer; + } + } + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + uint16_t i, j, k, l, m, n; + int conv_out; + signed char in_row, in_col; + + if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out; j++) + { + for (k = 0; k < dim_im_out; k++) + { + conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel; m++) + { + for (n = 0; n < dim_kernel; n++) + { + in_row = stride * j + m - padding; + in_col = stride * k + n - padding; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += + Im_in[(in_row * dim_im_in + in_col) * ch_im_in + + l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c new file mode 100644 index 0000000..0274202 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q15_fast.c + * Description: Fast Q15 version of convolution + * + * $Date: 24. May 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + + /** + * @brief Fast Q15 convolution function (non-sqaure shape) + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in_x input tensor dimention x + * @param[in] dim_im_in_y input tensor dimention y + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel_x filter kernel size x + * @param[in] dim_kernel_y filter kernel size y + * @param[in] padding_x padding size x + * @param[in] padding_y padding size y + * @param[in] stride_x convolution stride x + * @param[in] stride_y convolution stride y + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out_x output tensor dimension x + * @param[in] dim_im_out_y output tensor dimension y + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * @details + * + * Buffer size: + * + * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel + * + * bufferB size: 0 + * + * Input dimension constraints: + * + * ch_im_in is multiple of 2 + * + * ch_im_out is multipe of 2 + * + */ + +arm_status +arm_convolve_HWC_q15_fast_nonsquare(const q15_t * Im_in, + const uint16_t dim_im_in_x, + const uint16_t dim_im_in_y, + const uint16_t ch_im_in, + const q15_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel_x, + const uint16_t dim_kernel_y, + const uint16_t padding_x, + const uint16_t padding_y, + const uint16_t stride_x, + const uint16_t stride_y, + const q15_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q15_t * Im_out, + const uint16_t dim_im_out_x, + const uint16_t dim_im_out_y, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + q15_t *pBuffer = bufferA; + q15_t *im_buffer = bufferA; + q15_t *pOut = Im_out; + + if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + /* This part implements the im2col function */ + for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) + { + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++) + { + for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */ + memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (i_out_x & 0x1) + { + int i; + /* initialize the matrix pointers for A */ + const q15_t *pA = wt; + + /* set up the second output pointers */ + q15_t *pOut2 = pOut + ch_im_out; + + /* this loop over rows in A */ + for (i = 0; i < ch_im_out; i += 2) + { + /* setup pointers for B */ + q15_t *pB = im_buffer; + const q15_t *pB2 = pB + ch_im_in * dim_kernel_y * dim_kernel_x; + + /* aling the second pointer for A */ + const q15_t *pA2 = pA + ch_im_in * dim_kernel_y * dim_kernel_x; + + /* init the sum with bias */ + q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 1; + /* accumulate over the vector */ + while (colCnt) + { + q31_t inA1 = *__SIMD32(pA)++; + q31_t inB1 = *__SIMD32(pB)++; + q31_t inA2 = *__SIMD32(pA2)++; + q31_t inB2 = *__SIMD32(pB2)++; + + sum = __SMLAD(inA1, inB1, sum); + sum2 = __SMLAD(inA1, inB2, sum2); + sum3 = __SMLAD(inA2, inB1, sum3); + sum4 = __SMLAD(inA2, inB2, sum4); + + colCnt--; + } /* while over colCnt */ + colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x1; + while (colCnt) + { + q15_t inA1 = *pA++; + q15_t inB1 = *pB++; + q15_t inA2 = *pA2++; + q15_t inB2 = *pB2++; + + sum += inA1 * inB1; + sum2 += inA1 * inB2; + sum3 += inA2 * inB1; + sum4 += inA2 * inB2; + colCnt--; + } /* while over colCnt */ + *pOut++ = (q15_t) __SSAT(sum >> out_shift, 16); + *pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16); + *pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16); + *pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16); + + /* skip the row computed with A2 */ + pA += ch_im_in * dim_kernel_y * dim_kernel_x; + } /* for over ch_im_out */ + + pOut += ch_im_out; + /* counter reset */ + pBuffer = im_buffer; + } + } + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + uint16_t i, j, k, l, m, n; + int conv_out; + signed char in_row, in_col; + + if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out_y; j++) + { + for (k = 0; k < dim_im_out_x; k++) + { + conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel_y; m++) + { + for (n = 0; n < dim_kernel_x; n++) + { + in_row = stride_y * j + m - padding_y; + in_col = stride_x * k + n - padding_x; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += + Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + + l] * wt[i * ch_im_in * dim_kernel_x * dim_kernel_y + (m * dim_kernel_x + + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c new file mode 100644 index 0000000..42bfb1f --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c @@ -0,0 +1,279 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q7_RGB.c + * Description: Q7 version of convolution for RGB image + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + + /** + * @brief Q7 convolution function for RGB image + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * @details + * + * Buffer size: + * + * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel + * + * bufferB size: 0 + * + * Input dimension constraints: + * + * ch_im_in equals 3 + * + * This kernel is written exclusively for convolution with ch_im_in + * equals 3. This applies on the first layer of CNNs which has input + * image with RGB format. + */ + +arm_status +arm_convolve_HWC_q7_RGB(const q7_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, const uint16_t dim_im_out, q15_t * bufferA, q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + /* + * Here we use bufferA as q15_t internally as computation are done with q15_t level + * im2col are done to output in q15_t format from q7_t input + */ + q15_t *pBuffer = bufferA; + q7_t *pOut = Im_out; + + // check if number of input channels is 3 + if (ch_im_in != 3) + { + return ARM_MATH_SIZE_MISMATCH; + } + // This part implements the im2col function + for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* Equivalent to arm_fill_q15(0, pBuffer, ch_im_in) with assumption: ch_im_in = 3 */ + *__SIMD32(pBuffer) = 0x0; + *(pBuffer + 2) = 0; + pBuffer += 3; + } else + { + /* + * Equivalent to: + * arm_q7_to_q15_no_shift( (q7_t*)Im_in+(i_ker_y*dim_im_in+i_ker_x)*3, pBuffer, 3); + */ + + const q7_t *pPixel = Im_in + (i_ker_y * dim_im_in + i_ker_x) * 3; + q31_t buf = *__SIMD32(pPixel); + + union arm_nnword top; + union arm_nnword bottom; + + top.word = __SXTB16(buf); + bottom.word = __SXTB16(__ROR(buf, 8)); + +#ifndef ARM_MATH_BIG_ENDIAN + /* + * little-endian, | omit | 3rd | 2nd | 1st | + * MSB LSB + * top | 3rd | 1st |; bottom | omit | 2nd | + * + * version 1, need to swap 2nd and 3rd weight + * *__SIMD32(pBuffer) = top.word; + * *(pBuffer+2) = bottom.half_words[0]; + * + * version 2, no weight shuffling required + */ + *pBuffer++ = top.half_words[0]; + *__SIMD32(pBuffer) = __PKHBT(bottom.word, top.word, 0); +#else + /* + * big-endian, | 1st | 2nd | 3rd | omit | + * MSB LSB + * top | 2nd | omit |; bottom | 1st | 3rd | + * + * version 1, need to swap 2nd and 3rd weight + * *__SIMD32(pBuffer) = bottom.word; + * *(pBuffer+2) = top.half_words[1]; + * + * version 2, no weight shuffling required + */ + *pBuffer++ = bottom.half_words[0]; + *__SIMD32(pBuffer) = __PKHTB(top.word, bottom.word, 0); +#endif + pBuffer += 2; + } + } + } + + if (pBuffer == bufferA + 2 * 3 * dim_kernel * dim_kernel) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15(wt, bufferA, + ch_im_out, + 3 * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); + + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* left-over because odd number of output pixels */ + if (pBuffer != bufferA) + { + const q7_t *pA = wt; + int i; + + for (i = 0; i < ch_im_out; i++) + { + q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + q15_t *pB = bufferA; + /* basically each time it process 4 entries */ + uint16_t colCnt = 3 * dim_kernel * dim_kernel >> 2; + + while (colCnt) + { + + q31_t inA1, inA2; + q31_t inB1, inB2; + + pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2); + + inB1 = *__SIMD32(pB)++; + sum = __SMLAD(inA1, inB1, sum); + inB2 = *__SIMD32(pB)++; + sum = __SMLAD(inA2, inB2, sum); + + colCnt--; + } + colCnt = 3 * dim_kernel * dim_kernel & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + sum += inA1 * inB1; + colCnt--; + } + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + } + } +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + uint16_t i, j, k, l, m, n; + int conv_out; + signed char in_row, in_col; + + // check if number of input channels is 3 + if (ch_im_in != 3) + { + return ARM_MATH_SIZE_MISMATCH; + } + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out; j++) + { + for (k = 0; k < dim_im_out; k++) + { + conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel; m++) + { + for (n = 0; n < dim_kernel; n++) + { + /* if-for implementation */ + in_row = stride * j + m - padding; + in_col = stride * k + n - padding; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += + Im_in[(in_row * dim_im_in + in_col) * ch_im_in + + l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return (ARM_MATH_SUCCESS); +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c new file mode 100644 index 0000000..a926086 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q7_basic.c + * Description: Q7 version of convolution + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + + /** + * @brief Basic Q7 convolution function + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns ARM_MATH_SUCCESS + * + * @details + * + * Buffer size: + * + * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel + * + * bufferB size: 0 + * + * This basic version is designed to work for any input tensor and weight + * dimension. + */ + +arm_status +arm_convolve_HWC_q7_basic(const q7_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, + const uint16_t dim_im_out, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + /* + * Here we use bufferA as q15_t internally as computation are done with q15_t level + * im2col are done to output in q15_t format from q7_t input + */ + q15_t *pBuffer = bufferA; + q7_t *pOut = Im_out; + + /* This part implements the im2col function */ + for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* Filling 0 for out-of-bound paddings */ + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + /* Copying the pixel data to column */ + arm_q7_to_q15_no_shift((q7_t *) + Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + /* Computation is filed for every 2 columns */ + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15(wt, bufferA, + ch_im_out, + ch_im_in * + dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); + + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* left-over because odd number of output pixels */ + if (pBuffer != bufferA) + { + const q7_t *pA = wt; + int i; + + for (i = 0; i < ch_im_out; i++) + { + /* Load the accumulator with bias first */ + q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + + /* Point to the beging of the im2col buffer */ + q15_t *pB = bufferA; + + /* Each time it process 4 entries */ + uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2; + + while (colCnt) + { + q31_t inA1, inA2; + q31_t inB1, inB2; + + pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2); + + inB1 = *__SIMD32(pB)++; + sum = __SMLAD(inA1, inB1, sum); + inB2 = *__SIMD32(pB)++; + sum = __SMLAD(inA2, inB2, sum); + + colCnt--; + } + colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + sum += inA1 * inB1; + colCnt--; + } + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + } + } +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + uint16_t i, j, k, l, m, n; + int conv_out; + signed char in_row, in_col; + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out; j++) + { + for (k = 0; k < dim_im_out; k++) + { + conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel; m++) + { + for (n = 0; n < dim_kernel; n++) + { + // if-for implementation + in_row = stride * j + m - padding; + in_col = stride * k + n - padding; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += + Im_in[(in_row * dim_im_in + in_col) * ch_im_in + + l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c new file mode 100644 index 0000000..b426b92 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q7_basic.c + * Description: Q7 version of convolution + * + * $Date: 13. July 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + + /** + * @brief Basic Q7 convolution function (non-sqaure shape) + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in_x input tensor dimention x + * @param[in] dim_im_in_y input tensor dimention y + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel_x filter kernel size x + * @param[in] dim_kernel_y filter kernel size y + * @param[in] padding_x padding size x + * @param[in] padding_y padding size y + * @param[in] stride_x convolution stride x + * @param[in] stride_y convolution stride y + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out_x output tensor dimension x + * @param[in] dim_im_out_y output tensor dimension y + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns ARM_MATH_SUCCESS + */ + +arm_status arm_convolve_HWC_q7_basic_nonsquare(const q7_t * Im_in, + const uint16_t dim_im_in_x, + const uint16_t dim_im_in_y, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel_x, + const uint16_t dim_kernel_y, + const uint16_t padding_x, + const uint16_t padding_y, + const uint16_t stride_x, + const uint16_t stride_y, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, + const uint16_t dim_im_out_x, + const uint16_t dim_im_out_y, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + /* + * Here we use bufferA as q15_t internally as computation are done with q15_t level + * im2col are done to output in q15_t format from q7_t input + */ + q15_t *pBuffer = bufferA; + q7_t *pOut = Im_out; + + /* This part implements the im2col function */ + for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) + { + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++) + { + for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) + { + /* Filling 0 for out-of-bound paddings */ + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + /* Copying the pixel data to column */ + arm_q7_to_q15_no_shift((q7_t *) + Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + /* Computation is filed for every 2 columns */ + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_y * dim_kernel_x) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15(wt, bufferA, + ch_im_out, + ch_im_in * + dim_kernel_y * dim_kernel_x, bias_shift, out_shift, bias, pOut); + + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* left-over because odd number of output pixels */ + if (pBuffer != bufferA) + { + const q7_t *pA = wt; + int i; + + for (i = 0; i < ch_im_out; i++) + { + /* Load the accumulator with bias first */ + q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + + /* Point to the beging of the im2col buffer */ + q15_t *pB = bufferA; + + /* Each time it process 4 entries */ + uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 2; + + while (colCnt) + { + q31_t inA1, inA2; + q31_t inB1, inB2; + + pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2); + + inB1 = *__SIMD32(pB)++; + sum = __SMLAD(inA1, inB1, sum); + inB2 = *__SIMD32(pB)++; + sum = __SMLAD(inA2, inB2, sum); + + colCnt--; + } + colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + sum += inA1 * inB1; + colCnt--; + } + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + } + } +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + uint16_t i, j, k, l, m, n; + int conv_out; + signed char in_row, in_col; + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out_y; j++) + { + for (k = 0; k < dim_im_out_x; k++) + { + conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel_y; m++) + { + for (n = 0; n < dim_kernel_x; n++) + { + // if-for implementation + in_row = stride_y * j + m - padding_y; + in_col = stride_x * k + n - padding_x; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += + Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] * + wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + + (m * dim_kernel_x + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c new file mode 100644 index 0000000..7b59d79 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c @@ -0,0 +1,408 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q7_fast.c + * Description: Fast Q7 version of convolution + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + + /** + * @brief Fast Q7 convolution function + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * @details + * + * Buffer size: + * + * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel + * + * bufferB size: 0 + * + * Input dimension constraints: + * + * ch_im_in is multiple of 4 ( because of the SIMD32 read and swap ) + * + * ch_im_out is multipe of 2 ( bacause 2x2 mat_mult kernel ) + * + * The im2col converts the Q7 tensor input into Q15 column, which is stored in + * bufferA. There is reordering happenning during this im2col process with + * arm_q7_to_q15_reordered_no_shift. For every four elements, the second and + * third elements are swapped. + * + * The computation kernel arm_nn_mat_mult_kernel_q7_q15_reordered does the + * GEMM computation with the reordered columns. + * + * To speed-up the determination of the padding condition, we split the + * computation into 3x3 parts, i.e., {top, mid, bottom} X {left, mid, right}. + * This reduces the total number of boundary condition checks and improves + * the data copying performance. + */ + +arm_status +arm_convolve_HWC_q7_fast(const q7_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, + const uint16_t dim_im_out, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + /* + * Here we use bufferA as q15_t internally as computation are done with q15_t level + * im2col are done to output in q15_t format from q7_t input + */ + + q15_t *pBuffer = bufferA; + q7_t *pOut = Im_out; + + if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + /* + * Here we split the entire matrix into three regions depending on the padding situation + * Top: i_out_y from 0 to padding - 1 + * Middle: i_out_y from padding to dim_im_out-padding-1 + * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1 + */ + + /* top part */ + for (i_out_y = 0; i_out_y < padding; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift + ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, + bufferA, + ch_im_out, + ch_im_in + * + dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* middle part, here we also divide the x into left, mid and right */ + for (; i_out_y < dim_im_out - padding; i_out_y++) + { + + /* left part */ + for (i_out_x = 0; i_out_x < padding; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift + ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, + bufferA, + ch_im_out, + ch_im_in + * + dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + + /* mid part */ + for (; i_out_x < dim_im_out - padding; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + + + (i_ker_y * + dim_im_in + + i_out_x * + stride - padding) * ch_im_in, pBuffer, ch_im_in * dim_kernel); + pBuffer += ch_im_in * dim_kernel; + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, + bufferA, + ch_im_out, + ch_im_in + * + dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + + /* right part */ + for (; i_out_x < dim_im_out; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift + ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, + bufferA, + ch_im_out, + ch_im_in + * + dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + } + + for (; i_out_y < dim_im_out; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift + ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, + bufferA, + ch_im_out, + ch_im_in + * + dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* check if there is left-over for compute */ + if (pBuffer != bufferA) + { + const q7_t *pA = wt; + int i; + + for (i = 0; i < ch_im_out; i++) + { + q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); + q15_t *pB = bufferA; + /* each time it process 4 entries */ + uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2; + + while (colCnt) + { + + q31_t inA1, inA2; + q31_t inB1, inB2; + + pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA1, &inA2); + + inB1 = *__SIMD32(pB)++; + sum = __SMLAD(inA1, inB1, sum); + inB2 = *__SIMD32(pB)++; + sum = __SMLAD(inA2, inB2, sum); + + colCnt--; + } + colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + sum += inA1 * inB1; + colCnt--; + } + *pOut = (q7_t) __SSAT((sum >> out_shift), 8); + pOut++; + + } + + } +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + uint16_t i, j, k, l, m, n; + int conv_out; + signed char in_row, in_col; + + if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out; j++) + { + for (k = 0; k < dim_im_out; k++) + { + conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel; m++) + { + for (n = 0; n < dim_kernel; n++) + { + // if-for implementation + in_row = stride * j + m - padding; + in_col = stride * k + n - padding; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += + Im_in[(in_row * dim_im_in + in_col) * ch_im_in + + l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c new file mode 100644 index 0000000..f2aa4a2 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c @@ -0,0 +1,379 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_convolve_HWC_q7_fast_nonsquare.c + * Description: Fast Q7 version of convolution (non-sqaure shape) + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + +/** + * @brief Fast Q7 convolution function (non-sqaure shape) + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in_x input tensor dimention x + * @param[in] dim_im_in_y input tensor dimention y + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel_x filter kernel size x + * @param[in] dim_kernel_y filter kernel size y + * @param[in] padding_x padding size x + * @param[in] padding_y padding size y + * @param[in] stride_x convolution stride x + * @param[in] stride_y convolution stride y + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out_x output tensor dimension x + * @param[in] dim_im_out_y output tensor dimension y + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * This function is the version with full list of optimization tricks, but with + * some contraints: + * ch_im_in is multiple of 4 + * ch_im_out is multiple of 2 + */ + +arm_status arm_convolve_HWC_q7_fast_nonsquare(const q7_t * Im_in, + const uint16_t dim_im_in_x, + const uint16_t dim_im_in_y, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel_x, + const uint16_t dim_kernel_y, + const uint16_t padding_x, + const uint16_t padding_y, + const uint16_t stride_x, + const uint16_t stride_y, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, + const uint16_t dim_im_out_x, + const uint16_t dim_im_out_y, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; + + /* ----------------------- + * Here we use bufferA as q15_t internally as computation are done with q15_t level + * im2col are done to output in q15_t format from q7_t input + */ + + q15_t *pBuffer = bufferA; + q7_t *pOut = Im_out; + + if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + /* + * Here we split the entire matrix into three regions depending on the padding situation + * Top: i_out_y from 0 to padding - 1 + * Middle: i_out_y from padding to dim_im_out-padding-1 + * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1 + */ + + /* top part */ + for (i_out_y = 0; i_out_y < padding_y; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; + i_ker_y++) + { + for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; + i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, + pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, + bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* middle part, here we also divide the x into left, mid and right */ + for (; i_out_y < dim_im_out_y - padding_y; i_out_y++) + { + + /* left part */ + for (i_out_x = 0; i_out_x < padding_x; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; + i_ker_y++) + { + for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; + i_ker_x++) + { + if (i_ker_x < 0 || i_ker_x >= dim_im_in_x) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, + pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, + bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + + /* mid part */ + for (; i_out_x < dim_im_out_x - padding_x; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; + i_ker_y++) + { + arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + + (i_ker_y * dim_im_in_x + i_out_x * stride_x - padding_x) * ch_im_in, + pBuffer, ch_im_in * dim_kernel_x); + pBuffer += ch_im_in * dim_kernel_x; + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, + bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + + /* right part */ + for (; i_out_x < dim_im_out_x; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; + i_ker_y++) + { + for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; + i_ker_x++) + { + if (i_ker_x < 0 || i_ker_x >= dim_im_in_x) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, + pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, + bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + } + + for (; i_out_y < dim_im_out_y; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) + { + /* This part implements the im2col function */ + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; + i_ker_y++) + { + for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; + i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) + { + /* arm_fill_q15(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, sizeof(q15_t)*ch_im_in); + } else + { + arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, + pBuffer, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) + { + pOut = + arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, + bias_shift, out_shift, bias, pOut); + /* counter reset */ + pBuffer = bufferA; + } + } + } + + /* check if there is left-over for compute */ + if (pBuffer != bufferA) + { + const q7_t *pA = wt; + int i; + for (i = 0; i < ch_im_out; i++) + { + q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + q15_t *pB = bufferA; + /* basically each time it process 4 entries */ + uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2; + + while (colCnt) + { + + q31_t inA1, inA2; + q31_t inB1, inB2; + + pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2); + + inB1 = *__SIMD32(pB)++; + sum = __SMLAD(inA1, inB1, sum); + inB2 = *__SIMD32(pB)++; + sum = __SMLAD(inA2, inB2, sum); + + colCnt--; + } + colCnt = (ch_im_in * dim_kernel_y * dim_kernel_x) & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + sum += inA1 * inB1; + colCnt--; + } + *pOut = (q7_t) __SSAT((sum >> out_shift), 8); + pOut++; + + } + + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + int i, j, k, l, m, n; + int conv_out; + int in_row, in_col; + + if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) + { + /* check if the input dimension meets the constraints */ + return ARM_MATH_SIZE_MISMATCH; + } + + for (i = 0; i < ch_im_out; i++) + { + for (j = 0; j < dim_im_out_y; j++) + { + for (k = 0; k < dim_im_out_x; k++) + { + conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + for (m = 0; m < dim_kernel_y; m++) + { + for (n = 0; n < dim_kernel_x; n++) + { + /* if-for implementation */ + in_row = stride_y * j + m - padding_y; + in_col = stride_x * k + n - padding_x; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) + { + for (l = 0; l < ch_im_in; l++) + { + conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] * + wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_x + n) * ch_im_in + l]; + } + } + } + } + Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c b/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c new file mode 100644 index 0000000..68ebeb8 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c @@ -0,0 +1,418 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_depthwise_separable_conv_HWC_q7.c + * Description: Q7 depthwise separable convolution function + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + +/** + * @brief Q7 depthwise separable convolution function + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * @details + * + * Buffer size: + * + * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel + * + * bufferB size: 0 + * + * Input dimension constraints: + * + * ch_im_in equals ch_im_out + * + * Implementation: + * There are 3 nested loop here: + * Inner loop: calculate each output value with MAC instruction over an accumulator + * Mid loop: loop over different output channel + * Outer loop: loop over different output (x, y) + */ + +arm_status arm_depthwise_separable_conv_HWC_q7(const q7_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, + const uint16_t dim_im_out, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_out_y, i_out_x; + int16_t i_ker_y, i_ker_x; + q7_t *colBuffer = (q7_t *) bufferA; + q7_t *pBuffer = colBuffer; + const q7_t *pBias = bias; + q7_t *pOut = Im_out; + uint16_t rowCnt; + uint16_t row_shift; + + /* do some checking here, basically ch_im_in == ch_im_out */ + if (ch_im_in != ch_im_out) + { + return ARM_MATH_SIZE_MISMATCH; + } + + for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + /* we first do im2col here */ + for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) + { + for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) + { + /* arm_fill_q7(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, ch_im_in); + } else + { + /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */ + memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + /* we will do the computation here for each channel */ + rowCnt = ch_im_out >> 2; + row_shift = 0; + pBias = bias; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = (dim_kernel * dim_kernel) >> 1; + q7_t *pB = colBuffer + row_shift; + const q7_t *pA = wt + row_shift; + row_shift += 4; + +#ifdef USE_INTRINSIC + +#ifndef ARM_MATH_BIG_ENDIAN + + while (colCnt) + { + q31_t inA1, inA2, inB1, inB2, opA, opB; + + inB1 = *__SIMD32(pB); + pB += ch_im_in; + opB = *__SIMD32(pB); + pB += ch_im_in; + inB2 = __PKHTB(opB, inB1, 16); + inB1 = __PKHBT(inB1, opB, 16); + inA1 = *__SIMD32(pA); + pA += ch_im_in; + opB = *__SIMD32(pA); + pA += ch_im_in; + inA2 = __PKHTB(opB, inA1, 16); + inA1 = __PKHBT(inA1, opB, 16); + opA = __SXTB16(inA1); + opB = __SXTB16(inB1); + sum = __SMLAD(opA, opB, sum); + opA = __SXTB16(__ROR(inA1, 8)); + opB = __SXTB16(__ROR(inB1, 8)); + sum2 = __SMLAD(opA, opB, sum2); + opA = __SXTB16(inA2); + opB = __SXTB16(inB2); + sum3 = __SMLAD(opA, opB, sum3); + opA = __SXTB16(__ROR(inA2, 8)); + opB = __SXTB16(__ROR(inB2, 8)); + sum4 = __SMLAD(opA, opB, sum4); + colCnt--; + } +#else + + while (colCnt) + { + q31_t inA1, inA2, inB1, inB2, opA, opB; + + inB1 = *__SIMD32(pB); + pB += ch_im_in; + opB = *__SIMD32(pB); + pB += ch_im_in; + inB2 = __PKHBT(opB, inB1, 16); + inB1 = __PKHTB(inB1, opB, 16); + inA1 = *__SIMD32(pA); + pA += ch_im_in; + opB = *__SIMD32(pA); + pA += ch_im_in; + inA2 = __PKHBT(opB, inA1, 16); + inA1 = __PKHTB(inA1, opB, 16); + opA = __SXTB16(inA1); + opB = __SXTB16(inB1); + sum2 = __SMLAD(opA, opB, sum2); + opA = __SXTB16(__ROR(inA1, 8)); + opB = __SXTB16(__ROR(inB1, 8)); + sum = __SMLAD(opA, opB, sum); + opA = __SXTB16(inA2); + opB = __SXTB16(inB2); + sum4 = __SMLAD(opA, opB, sum4); + opA = __SXTB16(__ROR(inA2, 8)); + opB = __SXTB16(__ROR(inB2, 8)); + sum3 = __SMLAD(opA, opB, sum3); + colCnt--; + } + +#endif /* ARM_MATH_BIG_ENDIAN */ + +#else + +#ifndef ARM_MATH_BIG_ENDIAN + /* + * r0 r1 r2 r3 r4 r5 + * inA1, inA2, inB1, inB2, opA, opB + */ + + asm volatile ("COL_LOOP_%=:\n" + "ldr.w r2, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "ldr.w r5, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "pkhtb r3, r5, r2, ASR #16\n" + "pkhbt r2, r2, r5, LSL #16\n" + "ldr.w r0, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "ldr.w r5, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "pkhtb r1, r5, r0, ASR #16\n" + "pkhbt r0, r0, r5, LSL #16\n" + "sxtb16 r4, r0\n" + "sxtb16 r5, r2\n" + "smlad %[sum], r4, r5, %[sum]\n" + "mov.w r4, r0, ror #8\n" + "mov.w r5, r2, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum2], r4, r5, %[sum2]\n" + "sxtb16 r4, r1\n" + "sxtb16 r5, r3\n" + "smlad %[sum3], r4, r5, %[sum3]\n" + "mov.w r4, r1, ror #8\n" + "mov.w r5, r3, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum4], r4, r5, %[sum4]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP_%=\n":[sum] + "+r"(sum),[sum2] "+r"(sum2), + [sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB), + [pA] "+r"(pA):[colCnt] + "r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5"); +#else + /* + * r0 r1 r2 r3 r4 r5 + * inA1, inA2, inB1, inB2, opA, opB + */ + asm volatile ("COL_LOOP_%=:\n" + "ldr.w r2, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "ldr.w r5, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "pkhbt r3, r5, r2, LSL #16\n" + "pkhtb r2, r2, r5, ASR #16\n" + "ldr.w r0, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "ldr.w r5, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "pkhbt r1, r5, r0, LSL #16\n" + "pkhtb r0, r0, r5, ASR #16\n" + "sxtb16 r4, r0\n" + "sxtb16 r5, r2\n" + "smlad %[sum2], r4, r5, %[sum2]\n" + "mov.w r4, r0, ror #8\n" + "mov.w r5, r2, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum], r4, r5, %[sum]\n" + "sxtb16 r4, r1\n" + "sxtb16 r5, r3\n" + "smlad %[sum4], r4, r5, %[sum4]\n" + "mov.w r4, r1, ror #8\n" + "mov.w r5, r3, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum3], r4, r5, %[sum3]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP_%=\n":[sum] + "+r"(sum),[sum2] "+r"(sum2), + [sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB), + [pA] "+r"(pA):[colCnt] + "r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5"); + +#endif /* ARM_MATH_BIG_ENDIAN */ + +#endif /* USE_INTRINSIC */ + + colCnt = (dim_kernel * dim_kernel) & 0x1; + while (colCnt) + { + union arm_nnword inA, inB; + inA.word = *__SIMD32(pA); + pA += ch_im_in; + inB.word = *__SIMD32(pB); + pB += ch_im_in; + sum += inA.bytes[0] * inB.bytes[0]; + sum2 += inA.bytes[1] * inB.bytes[1]; + sum3 += inA.bytes[2] * inB.bytes[2]; + sum4 += inA.bytes[3] * inB.bytes[3]; + colCnt--; + } + + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8); + + rowCnt--; + } + + rowCnt = ch_im_out & 0x3; + while (rowCnt) + { + q7_t *pB = colBuffer + row_shift; + const q7_t *pA = wt + row_shift; + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + uint16_t colCnt = (dim_kernel * dim_kernel); + + row_shift += 1; + + while (colCnt) + { + q7_t A1 = *pA; + q7_t B1 = *pB; + pA += ch_im_in; + pB += ch_im_in; + sum += A1 * B1; + + colCnt--; + } + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + rowCnt--; + } + + /* clear counter and pointers */ + pBuffer = colBuffer; + } + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + int i_out_y, i_out_x, i_ch_out, i_ker_x, i_ker_y; + int conv_out; + + /* do some checking here, basically ch_im_in == ch_im_out */ + if (ch_im_in != ch_im_out) + { + return ARM_MATH_SIZE_MISMATCH; + } + + for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) + { + for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++) + { + // for each output + conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift); + for (i_ker_y = 0; i_ker_y < dim_kernel; i_ker_y++) + { + for (i_ker_x = 0; i_ker_x < dim_kernel; i_ker_x++) + { + int in_row = stride * i_out_y + i_ker_y - padding; + int in_col = stride * i_out_x + i_ker_x - padding; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) + { + conv_out += + Im_in[(in_row * + dim_im_in + + in_col) * + ch_im_in + + i_ch_out] * wt[(i_ker_y * dim_kernel + i_ker_x) * ch_im_out + i_ch_out]; + } + } + } + Im_out[(i_out_y * dim_im_out + + i_out_x) * ch_im_out + i_ch_out] = (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return ARM_MATH_SUCCESS; + +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c b/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c new file mode 100644 index 0000000..397f233 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c @@ -0,0 +1,411 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_depthwise_separable_conv_HWC_q7_nonsquare.c + * Description: Q7 depthwise separable convolution function (non-square shape) + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup NNConv + * @{ + */ + +/** + * @brief Q7 depthwise separable convolution function (non-square shape) + * @param[in] Im_in pointer to input tensor + * @param[in] dim_im_in_x input tensor dimention x + * @param[in] dim_im_in_y input tensor dimention y + * @param[in] ch_im_in number of input tensor channels + * @param[in] wt pointer to kernel weights + * @param[in] ch_im_out number of filters, i.e., output tensor channels + * @param[in] dim_kernel_x filter kernel size x + * @param[in] dim_kernel_y filter kernel size y + * @param[in] padding_x padding sizes x + * @param[in] padding_y padding sizes y + * @param[in] stride_x convolution stride x + * @param[in] stride_y convolution stride y + * @param[in] bias pointer to bias + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in,out] Im_out pointer to output tensor + * @param[in] dim_im_out_x output tensor dimension x + * @param[in] dim_im_out_y output tensor dimension y + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] bufferB pointer to buffer space for output + * @return The function returns either + * ARM_MATH_SIZE_MISMATCH or ARM_MATH_SUCCESS based on the outcome of size checking. + * + * This function is the version with full list of optimization tricks, but with + * some contraints: + * ch_im_in is multiple of 2 + * ch_im_out is multiple of 2 + */ + +arm_status arm_depthwise_separable_conv_HWC_q7_nonsquare(const q7_t * Im_in, + const uint16_t dim_im_in_x, + const uint16_t dim_im_in_y, + const uint16_t ch_im_in, + const q7_t * wt, + const uint16_t ch_im_out, + const uint16_t dim_kernel_x, + const uint16_t dim_kernel_y, + const uint16_t padding_x, + const uint16_t padding_y, + const uint16_t stride_x, + const uint16_t stride_y, + const q7_t * bias, + const uint16_t bias_shift, + const uint16_t out_shift, + q7_t * Im_out, + const uint16_t dim_im_out_x, + const uint16_t dim_im_out_y, + q15_t * bufferA, + q7_t * bufferB) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + +/* + * Implementation: + * There are 3 nested loop here: + * Inner loop: calculate each output value with MAC instruction over an accumulator + * Mid loop: loop over different output channel + * Outer loop: loop over different output (x, y) + * + */ + + int16_t i_out_y, i_out_x; + int16_t i_ker_y, i_ker_x; + q7_t *colBuffer = (q7_t *) bufferA; + q7_t *pBuffer = colBuffer; + const q7_t *pBias = bias; + q7_t *pOut = Im_out; + uint16_t rowCnt; + uint16_t row_shift; + + /* do some checking here, basically ch_im_in == ch_im_out */ + if (ch_im_in != ch_im_out) + { + return ARM_MATH_SIZE_MISMATCH; + } + + for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) + { + /* we first do im2col here */ + for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; + i_ker_y++) + { + for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; + i_ker_x++) + { + if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) + { + /* arm_fill_q7(0, pBuffer, ch_im_in); */ + memset(pBuffer, 0, ch_im_in); + } else + { + /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */ + memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, ch_im_in); + } + pBuffer += ch_im_in; + } + } + + /* we will do the computation here for each channel */ + rowCnt = ch_im_out >> 2; + row_shift = 0; + pBias = bias; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = (dim_kernel_x * dim_kernel_y) >> 1; + q7_t *pB = colBuffer + row_shift; + const q7_t *pA = wt + row_shift; + row_shift += 4; + +#ifdef USE_INTRINSIC + +#ifndef ARM_MATH_BIG_ENDIAN + + while (colCnt) + { + q31_t inA1, inA2, inB1, inB2, opA, opB; + + inB1 = *__SIMD32(pB); + pB += ch_im_in; + opB = *__SIMD32(pB); + pB += ch_im_in; + inB2 = __PKHTB(opB, inB1, 16); + inB1 = __PKHBT(inB1, opB, 16); + inA1 = *__SIMD32(pA); + pA += ch_im_in; + opB = *__SIMD32(pA); + pA += ch_im_in; + inA2 = __PKHTB(opB, inA1, 16); + inA1 = __PKHBT(inA1, opB, 16); + opA = __SXTB16(inA1); + opB = __SXTB16(inB1); + sum = __SMLAD(opA, opB, sum); + opA = __SXTB16(__ROR(inA1, 8)); + opB = __SXTB16(__ROR(inB1, 8)); + sum2 = __SMLAD(opA, opB, sum2); + opA = __SXTB16(inA2); + opB = __SXTB16(inB2); + sum3 = __SMLAD(opA, opB, sum3); + opA = __SXTB16(__ROR(inA2, 8)); + opB = __SXTB16(__ROR(inB2, 8)); + sum4 = __SMLAD(opA, opB, sum4); + colCnt--; + } +#else + + while (colCnt) + { + q31_t inA1, inA2, inB1, inB2, opA, opB; + + inB1 = *__SIMD32(pB); + pB += ch_im_in; + opB = *__SIMD32(pB); + pB += ch_im_in; + inB2 = __PKHBT(opB, inB1, 16); + inB1 = __PKHTB(inB1, opB, 16); + inA1 = *__SIMD32(pA); + pA += ch_im_in; + opB = *__SIMD32(pA); + pA += ch_im_in; + inA2 = __PKHBT(opB, inA1, 16); + inA1 = __PKHTB(inA1, opB, 16); + opA = __SXTB16(inA1); + opB = __SXTB16(inB1); + sum2 = __SMLAD(opA, opB, sum2); + opA = __SXTB16(__ROR(inA1, 8)); + opB = __SXTB16(__ROR(inB1, 8)); + sum = __SMLAD(opA, opB, sum); + opA = __SXTB16(inA2); + opB = __SXTB16(inB2); + sum4 = __SMLAD(opA, opB, sum4); + opA = __SXTB16(__ROR(inA2, 8)); + opB = __SXTB16(__ROR(inB2, 8)); + sum3 = __SMLAD(opA, opB, sum3); + colCnt--; + } + +#endif /* ARM_MATH_BIG_ENDIAN */ + +#else + +#ifndef ARM_MATH_BIG_ENDIAN + // r0 r1 r2 r3 r4 r5 + // inA1, inA2, inB1, inB2, opA, opB + asm volatile ("COL_LOOP:\n" + "ldr.w r2, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "ldr.w r5, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "pkhtb r3, r5, r2, ASR #16\n" + "pkhbt r2, r2, r5, LSL #16\n" + "ldr.w r0, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "ldr.w r5, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "pkhtb r1, r5, r0, ASR #16\n" + "pkhbt r0, r0, r5, LSL #16\n" + "sxtb16 r4, r0\n" + "sxtb16 r5, r2\n" + "smlad %[sum], r4, r5, %[sum]\n" + "mov.w r4, r0, ror #8\n" + "mov.w r5, r2, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum2], r4, r5, %[sum2]\n" + "sxtb16 r4, r1\n" + "sxtb16 r5, r3\n" + "smlad %[sum3], r4, r5, %[sum3]\n" + "mov.w r4, r1, ror #8\n" + "mov.w r5, r3, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum4], r4, r5, %[sum4]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt), + [ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5"); +#else + // r0 r1 r2 r3 r4 r5 + // inA1, inA2, inB1, inB2, opA, opB + asm volatile ("COL_LOOP:\n" + "ldr.w r2, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "ldr.w r5, [%[pB], #0]\n" + "add.w %[pB], %[pB], %[ch_im_in]\n" + "pkhbt r3, r5, r2, LSL #16\n" + "pkhtb r2, r2, r5, ASR #16\n" + "ldr.w r0, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "ldr.w r5, [%[pA], #0]\n" + "add.w %[pA], %[pA], %[ch_im_in]\n" + "pkhbt r1, r5, r0, LSL #16\n" + "pkhtb r0, r0, r5, ASR #16\n" + "sxtb16 r4, r0\n" + "sxtb16 r5, r2\n" + "smlad %[sum2], r4, r5, %[sum2]\n" + "mov.w r4, r0, ror #8\n" + "mov.w r5, r2, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum], r4, r5, %[sum]\n" + "sxtb16 r4, r1\n" + "sxtb16 r5, r3\n" + "smlad %[sum4], r4, r5, %[sum4]\n" + "mov.w r4, r1, ror #8\n" + "mov.w r5, r3, ror #8\n" + "sxtb16 r4, r4\n" + "sxtb16 r5, r5\n" + "smlad %[sum3], r4, r5, %[sum3]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt), + [ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5"); +#endif /*ARM_MATH_BIG_ENDIAN */ + +#endif /* USE_INTRINSIC */ + + colCnt = (dim_kernel_x * dim_kernel_y) & 0x1; + while (colCnt) + { + union arm_nnword inA, inB; + inA.word = *__SIMD32(pA); + pA += ch_im_in; + inB.word = *__SIMD32(pB); + pB += ch_im_in; + sum += inA.bytes[0] * inB.bytes[0]; + sum2 += inA.bytes[1] * inB.bytes[1]; + sum3 += inA.bytes[2] * inB.bytes[2]; + sum4 += inA.bytes[3] * inB.bytes[3]; + colCnt--; + } + + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8); + + rowCnt--; + } + + rowCnt = ch_im_out & 0x3; + while (rowCnt) + { + q7_t *pB = colBuffer + row_shift; + const q7_t *pA = wt + row_shift; + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + uint16_t colCnt = (dim_kernel_x * dim_kernel_y); + + row_shift += 1; + + while (colCnt) + { + q7_t A1 = *pA; + q7_t B1 = *pB; + pA += ch_im_in; + pB += ch_im_in; + sum += A1 * B1; + + colCnt--; + } + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + rowCnt--; + } + + // clear counter and pointers + pBuffer = colBuffer; + } + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + int i_out_y, i_out_x, i_ch_out; + int i_ker_y, i_ker_x; + + /* do some checking here, basically ch_im_in == ch_im_out */ + if (ch_im_in != ch_im_out) + { + return ARM_MATH_SIZE_MISMATCH; + } + + for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) + { + for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) + { + for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++) + { + // for each output + int conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift); + for (i_ker_y = 0; i_ker_y < dim_kernel_y; i_ker_y++) + { + for (i_ker_x = 0; i_ker_x < dim_kernel_x; i_ker_x++) + { + int in_row = stride_y * i_out_y + i_ker_y - padding_y; + int in_col = stride_x * i_out_x + i_ker_x - padding_x; + if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) + { + conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + i_ch_out] * + wt[(i_ker_y * dim_kernel_x + i_ker_x) * ch_im_out + i_ch_out]; + } + } + } + Im_out[(i_out_y * dim_im_out_x + i_out_x) * ch_im_out + i_ch_out] = + (q7_t) __SSAT((conv_out >> out_shift), 8); + } + } + } + +#endif /* ARM_MATH_DSP */ + + + /* Return to application */ + return ARM_MATH_SUCCESS; + +} + +/** + * @} end of NNConv group + */ diff --git a/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c b/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c new file mode 100644 index 0000000..a4adc5d --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c @@ -0,0 +1,187 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_nn_mat_mult_kernel_q7_q15.c + * Description: Matrix-multiplication function for convolution + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + + /** + * @brief Matrix-multiplication function for convolution + * @param[in] pA pointer to operand A + * @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors + * @param[in] ch_im_out numRow of A + * @param[in] numCol_A numCol of A + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias the bias + * @param[in,out] pOut pointer to output + * @return The function returns the incremented output pointer + * + * @details + * + * This function does the matrix multiplication with weight matrix + * and 2 columns from im2col. + */ + +q7_t *arm_nn_mat_mult_kernel_q7_q15(const q7_t * pA, + const q15_t * pInBuffer, + const uint16_t ch_im_out, + const uint16_t numCol_A, + const uint16_t bias_shift, + const uint16_t out_shift, + const q7_t * bias, + q7_t * pOut) +{ +#if defined (ARM_MATH_DSP) + /* set up the second output pointers */ + q7_t *pOut2 = pOut + ch_im_out; + const q7_t *pBias = bias; + + uint16_t rowCnt = ch_im_out >> 1; + /* this loop over rows in A */ + while (rowCnt) + { + /* setup pointers for B */ + const q15_t *pB = pInBuffer; + const q15_t *pB2 = pB + numCol_A; + + /* align the second pointer for A */ + const q7_t *pA2 = pA + numCol_A; + + /* init the sum with bias */ + q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = numCol_A >> 2; + /* accumulate over the vector */ + while (colCnt) + { + q31_t inA11, inA12, inA21, inA22; + q31_t inB1 = *__SIMD32(pB)++; + q31_t inB2 = *__SIMD32(pB2)++; + + pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12); + pA2 = (q7_t *) read_and_pad((void *)pA2, &inA21, &inA22); + + sum = __SMLAD(inA11, inB1, sum); + sum2 = __SMLAD(inA11, inB2, sum2); + sum3 = __SMLAD(inA21, inB1, sum3); + sum4 = __SMLAD(inA21, inB2, sum4); + + inB1 = *__SIMD32(pB)++; + inB2 = *__SIMD32(pB2)++; + + sum = __SMLAD(inA12, inB1, sum); + sum2 = __SMLAD(inA12, inB2, sum2); + sum3 = __SMLAD(inA22, inB1, sum3); + sum4 = __SMLAD(inA22, inB2, sum4); + + colCnt--; + } /* while over colCnt */ + colCnt = numCol_A & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + q7_t inA2 = *pA2++; + q15_t inB2 = *pB2++; + + sum += inA1 * inB1; + sum2 += inA1 * inB2; + sum3 += inA2 * inB1; + sum4 += inA2 * inB2; + colCnt--; + } /* while over colCnt */ + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8); + *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8); + *pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8); + + /* skip the row computed with A2 */ + pA += numCol_A; + rowCnt--; + } /* for over ch_im_out */ + + /* compute left-over row if any */ + if (ch_im_out & 0x1) + { + /* setup pointers for B */ + const q15_t *pB = pInBuffer; + const q15_t *pB2 = pB + numCol_A; + + /* load the bias */ + q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = numCol_A >> 2; + while (colCnt) + { + q31_t inA11, inA12; + q31_t inB1 = *__SIMD32(pB)++; + q31_t inB2 = *__SIMD32(pB2)++; + + pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12); + + sum = __SMLAD(inA11, inB1, sum); + sum2 = __SMLAD(inA11, inB2, sum2); + + inB1 = *__SIMD32(pB)++; + inB2 = *__SIMD32(pB2)++; + sum = __SMLAD(inA12, inB1, sum); + sum2 = __SMLAD(inA12, inB2, sum2); + + colCnt--; + } + colCnt = numCol_A & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + q15_t inB2 = *pB2++; + + sum += inA1 * inB1; + sum2 += inA1 * inB2; + colCnt--; + } + + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8); + } + + pOut += ch_im_out; + + /* return the new output pointer with offset */ + return pOut; +#else + /* To be completed */ + return NULL; +#endif /* ARM_MATH_DSP */ + +} diff --git a/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c b/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c new file mode 100644 index 0000000..deef7c6 --- /dev/null +++ b/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_nn_mat_mult_kernel_q7_q15_reordered.c + * Description: Matrix-multiplication function for convolution with reordered columns + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * -------------------------------------------------------------------- */ + +#include "arm_nnfunctions.h" +#include "arm_math.h" + + /** + * @brief Matrix-multiplication function for convolution with reordered columns + * @param[in] pA pointer to operand A + * @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors + * @param[in] ch_im_out numRow of A + * @param[in] numCol_A numCol of A + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias the bias + * @param[in,out] pOut pointer to output + * @return The function returns the incremented output pointer + * + * @details + * + * This function assumes that data in pInBuffer are reordered + */ + +q7_t *arm_nn_mat_mult_kernel_q7_q15_reordered(const q7_t * pA, + const q15_t * pInBuffer, + const uint16_t ch_im_out, + const uint16_t numCol_A, + const uint16_t bias_shift, + const uint16_t out_shift, + const q7_t * bias, + q7_t * pOut) +{ + +#if defined (ARM_MATH_DSP) + /* set up the second output pointers */ + q7_t *pOut2 = pOut + ch_im_out; + int i; + + /* this loop over rows in A */ + for (i = 0; i < ch_im_out; i += 2) + { + /* setup pointers for B */ + const q15_t *pB = pInBuffer; + const q15_t *pB2 = pB + numCol_A; + + /* align the second pointer for A */ + const q7_t *pA2 = pA + numCol_A; + + /* init the sum with bias */ + q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = numCol_A >> 2; + /* accumulate over the vector */ + while (colCnt) + { + q31_t inA11, inA12, inA21, inA22; + q31_t inB1 = *__SIMD32(pB)++; + q31_t inB2 = *__SIMD32(pB2)++; + + pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA11, &inA12); + pA2 = (q7_t *) read_and_pad_reordered((void *)pA2, &inA21, &inA22); + + sum = __SMLAD(inA11, inB1, sum); + sum2 = __SMLAD(inA11, inB2, sum2); + sum3 = __SMLAD(inA21, inB1, sum3); + sum4 = __SMLAD(inA21, inB2, sum4); + + inB1 = *__SIMD32(pB)++; + inB2 = *__SIMD32(pB2)++; + + sum = __SMLAD(inA12, inB1, sum); + sum2 = __SMLAD(inA12, inB2, sum2); + sum3 = __SMLAD(inA22, inB1, sum3); + sum4 = __SMLAD(inA22, inB2, sum4); + + colCnt--; + } /* while over colCnt */ + colCnt = numCol_A & 0x3; + while (colCnt) + { + q7_t inA1 = *pA++; + q15_t inB1 = *pB++; + q7_t inA2 = *pA2++; + q15_t inB2 = *pB2++; + + sum += inA1 * inB1; + sum2 += inA1 * inB2; + sum3 += inA2 * inB1; + sum4 += inA2 * inB2; + colCnt--; + } /* while over colCnt */ + *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8); + *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8); + *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8); + *pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8); + + /* skip the row computed with A2 */ + pA += numCol_A; + } /* for over ch_im_out */ + + pOut += ch_im_out; + + /* return the new output pointer with offset */ + return pOut; +#else + /* To be completed */ + return NULL; +#endif /* ARM_MATH_DSP */ +} diff --git a/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c b/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c new file mode 100644 index 0000000..2746967 --- /dev/null +++ b/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_fully_connected_mat_q7_vec_q15.c + * Description: Mixed Q15-Q7 fully-connected layer function + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup FC + * @{ + */ + + /** + * @brief Mixed Q15-Q7 fully-connected layer function + * @param[in] pV pointer to input vector + * @param[in] pM pointer to matrix weights + * @param[in] dim_vec length of the vector + * @param[in] num_of_rows number of rows in weight matrix + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias pointer to bias + * @param[in,out] pOut pointer to output vector + * @param[in,out] vec_buffer pointer to buffer space for input + * @return The function returns ARM_MATH_SUCCESS + * + * @details + * + * Buffer size: + * + * vec_buffer size: 0 + * + * Q7_Q15 version of the fully connected layer + * + * Weights are in q7_t and Activations are in q15_t + * + */ + +arm_status +arm_fully_connected_mat_q7_vec_q15(const q15_t * pV, + const q7_t * pM, + const uint16_t dim_vec, + const uint16_t num_of_rows, + const uint16_t bias_shift, + const uint16_t out_shift, + const q7_t * bias, + q15_t * pOut, + q15_t * vec_buffer) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + const q7_t *pB = pM; + const q7_t *pB2; + q15_t *pO = pOut; + const q7_t *pBias = bias; + const q15_t *pA = pV; + + uint16_t rowCnt = num_of_rows >> 1; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + uint16_t colCnt = dim_vec >> 2; + + pA = pV; + pB2 = pB + dim_vec; + + while (colCnt) + { + q31_t inV, inM11, inM12, inM21, inM22; + pB = (q7_t *) read_and_pad((void *)pB, &inM11, &inM12); + pB2 = (q7_t *) read_and_pad((void *)pB2, &inM21, &inM22); + + inV = *__SIMD32(pA)++; + + sum = __SMLAD(inV, inM11, sum); + sum2 = __SMLAD(inV, inM21, sum2); + + inV = *__SIMD32(pA)++; + + sum = __SMLAD(inV, inM12, sum); + sum2 = __SMLAD(inV, inM22, sum2); + + colCnt--; + } + colCnt = dim_vec & 0x3; + while (colCnt) + { + q15_t inV = *pA++; + q7_t inM = *pB++; + q7_t inM2 = *pB2++; + + sum += inV * inM; + sum2 += inV * inM2; + colCnt--; + } /* while over colCnt */ + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum2 >> out_shift), 16)); + + /*adjust the pointers and counters */ + pB += dim_vec; + rowCnt--; + } + + /* left-over part of the rows */ + rowCnt = num_of_rows & 0x1; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + uint16_t colCnt = dim_vec >> 2; + + pA = pV; + + while (colCnt) + { + q31_t inV1, inV2, inM11, inM12; + + pB = (q7_t *) read_and_pad((void *)pB, &inM11, &inM12); + + inV1 = *__SIMD32(pA)++; + sum = __SMLAD(inV1, inM11, sum); + + inV2 = *__SIMD32(pA)++; + sum = __SMLAD(inV2, inM12, sum); + + colCnt--; + } + + /* left-over of the vector */ + colCnt = dim_vec & 0x3; + while (colCnt) + { + q15_t inV = *pA++; + q7_t inM = *pB++; + sum += inV * inM; + colCnt--; + } + + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + + rowCnt--; + } + +#else + int i, j; + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + for (i = 0; i < num_of_rows; i++) + { + int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + for (j = 0; j < dim_vec; j++) + { + ip_out += pV[j] * pM[i * dim_vec + j]; + } + pOut[i] = (q15_t) __SSAT((ip_out >> out_shift), 16); + } + +#endif /* ARM_MATH_DSP */ + + /* Return to ARM_MATH_SUCCESS */ + return (ARM_MATH_SUCCESS); + +} + +/** + * @} end of FC group + */ diff --git a/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c b/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c new file mode 100644 index 0000000..7be156f --- /dev/null +++ b/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c @@ -0,0 +1,403 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_fully_connected_mat_q7_vec_q15_opt.c + * Description: Mixed Q15-Q7 opt fully-connected layer function + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup FC + * @{ + */ + + /** + * @brief Mixed Q15-Q7 opt fully-connected layer function + * @param[in] pV pointer to input vector + * @param[in] pM pointer to matrix weights + * @param[in] dim_vec length of the vector + * @param[in] num_of_rows number of rows in weight matrix + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias pointer to bias + * @param[in,out] pOut pointer to output vector + * @param[in,out] vec_buffer pointer to buffer space for input + * @return The function returns ARM_MATH_SUCCESS + * + * @details + * + * Buffer size: + * + * vec_buffer size: 0 + * + * Q7_Q15 version of the fully connected layer + * + * Weights are in q7_t and Activations are in q15_t + * + * Limitation: x4 version requires weight reordering to work + * + * Here we use only one pointer to read 4 rows in the weight + * matrix. So if the original q7_t matrix looks like this: + * + * | a11 | a12 | a13 | a14 | a15 | a16 | a17 | + * + * | a21 | a22 | a23 | a24 | a25 | a26 | a27 | + * + * | a31 | a32 | a33 | a34 | a35 | a36 | a37 | + * + * | a41 | a42 | a43 | a44 | a45 | a46 | a47 | + * + * | a51 | a52 | a53 | a54 | a55 | a56 | a57 | + * + * | a61 | a62 | a63 | a64 | a65 | a66 | a67 | + * + * We operates on multiple-of-4 rows, so the first four rows becomes + * + * | a11 | a21 | a12 | a22 | a31 | a41 | a32 | a42 | + * + * | a13 | a23 | a14 | a24 | a33 | a43 | a34 | a44 | + * + * | a15 | a25 | a16 | a26 | a35 | a45 | a36 | a46 | + * + * The column left over will be in-order. + * which is: + * | a17 | a27 | a37 | a47 | + * + * For the left-over rows, we do 1x1 computation, so the data remains + * as its original order. + * + * So the stored weight matrix looks like this: + * + * | a11 | a21 | a12 | a22 | a31 | a41 | + * + * | a32 | a42 | a13 | a23 | a14 | a24 | + * + * | a33 | a43 | a34 | a44 | a15 | a25 | + * + * | a16 | a26 | a35 | a45 | a36 | a46 | + * + * | a17 | a27 | a37 | a47 | a51 | a52 | + * + * | a53 | a54 | a55 | a56 | a57 | a61 | + * + * | a62 | a63 | a64 | a65 | a66 | a67 | + * + */ + +arm_status +arm_fully_connected_mat_q7_vec_q15_opt(const q15_t * pV, + const q7_t * pM, + const uint16_t dim_vec, + const uint16_t num_of_rows, + const uint16_t bias_shift, + const uint16_t out_shift, const q7_t * bias, q15_t * pOut, q15_t * vec_buffer) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + const q7_t *pB = pM; + q15_t *pO = pOut; + const q7_t *pBias = bias; + const q15_t *pA = pV; + + uint16_t rowCnt = num_of_rows >> 2; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 1; + + pA = pV; + +#ifdef USE_INTRINSIC + +#ifndef ARM_MATH_BIG_ENDIAN + + while (colCnt) + { + q31_t inM11, inM12, inM13, inM14; + q31_t inV; + + inV = *__SIMD32(pA)++; + inM11 = *__SIMD32(pB)++; + inM12 = __SXTB16(__ROR(inM11, 8)); + inM11 = __SXTB16(inM11); + sum = __SMLAD(inM11, inV, sum); + sum2 = __SMLAD(inM12, inV, sum2); + inM13 = *__SIMD32(pB)++; + inM14 = __SXTB16(__ROR(inM13, 8)); + inM13 = __SXTB16(inM13); + sum3 = __SMLAD(inM13, inV, sum3); + sum4 = __SMLAD(inM14, inV, sum4); + colCnt--; + } + +#else + + while (colCnt) + { + q31_t inM11, inM12, inM13, inM14; + q31_t inV; + + inV = *__SIMD32(pA)++; + inM11 = *__SIMD32(pB)++; + inM12 = __SXTB16(__ROR(inM11, 8)); + inM11 = __SXTB16(inM11); + sum = __SMLAD(inM12, inV, sum); + sum2 = __SMLAD(inM11, inV, sum2); + inM13 = *__SIMD32(pB)++; + inM14 = __SXTB16(__ROR(inM13, 8)); + inM13 = __SXTB16(inM13); + sum3 = __SMLAD(inM14, inV, sum3); + sum4 = __SMLAD(inM13, inV, sum4); + colCnt--; + } + +#endif /* ARM_MATH_BIG_ENDIAN */ + +#else + + /* + * register needed: + * loop counter: colCnt + * accumulators: sum, sum2, sum3, sum4 + * pointers: pB, pA + * weight data: inM11, inM12, inM13, inM14 + * activation data: inV + */ + +#ifndef ARM_MATH_BIG_ENDIAN + asm volatile ("COL_LOOP_%=:\n" + "ldr.w r4, [%[pA]], #4\n" + "ldr.w r1, [%[pB]], #8\n" + "mov.w r0, r1, ror #8\n" + "sxtb16 r0, r0\n" + "sxtb16 r1, r1\n" + "smlad %[sum], r4, r1, %[sum]\n" + "smlad %[sum2], r4, r0, %[sum2]\n" + "ldr.w r3, [%[pB], #-4]\n" + "mov.w r2, r3, ror #8\n" + "sxtb16 r2, r2\n" + "sxtb16 r3, r3\n" + "smlad %[sum3], r4, r3, %[sum3]\n" + "smlad %[sum4], r4, r2, %[sum4]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP_%=\n":[sum] "+r"(sum), + [sum2] "+r"(sum2),[sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4"); +#else + asm volatile ("COL_LOOP_%=:\n" + "ldr.w r4, [%[pA]], #4\n" + "ldr.w r1, [%[pB]], #8\n" + "mov.w r0, r1, ror #8\n" + "sxtb16 r0, r0\n" + "sxtb16 r1, r1\n" + "smlad %[sum], r4, r0, %[sum]\n" + "smlad %[sum2], r4, r1, %[sum2]\n" + "ldr.w r3, [%[pB], #-4]\n" + "mov.w r2, r3, ror #8\n" + "sxtb16 r2, r2\n" + "sxtb16 r3, r3\n" + "smlad %[sum3], r4, r2, %[sum3]\n" + "smlad %[sum4], r4, r3, %[sum4]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP_%=\n":[sum] "+r"(sum), + [sum2] "+r"(sum2),[sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4"); +#endif /* ARM_MATH_BIG_ENDIAN */ + +#endif /* USE_INTRINSIC */ + + colCnt = dim_vec & 0x1; + while (colCnt) + { + q15_t inV = *pA++; + q7_t inM = *pB++; + q7_t inM2 = *pB++; + q7_t inM3 = *pB++; + q7_t inM4 = *pB++; + + sum += inV * inM; + sum2 += inV * inM2; + sum3 += inV * inM3; + sum4 += inV * inM4; + colCnt--; + } /* while over colCnt */ + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum2 >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum3 >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum4 >> out_shift), 16)); + + /* adjust the pointers and counters */ + rowCnt--; + } + + /* left-over part of the rows */ + rowCnt = num_of_rows & 0x3; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 2; + + pA = pV; + + while (colCnt) + { + q31_t inV1, inV2, inM11, inM12; + + pB = (q7_t *) read_and_pad((void *)pB, &inM11, &inM12); + + inV1 = *__SIMD32(pA)++; + sum = __SMLAD(inV1, inM11, sum); + + inV2 = *__SIMD32(pA)++; + sum = __SMLAD(inV2, inM12, sum); + + colCnt--; + } + + /* left-over of the vector */ + colCnt = dim_vec & 0x3; + while (colCnt) + { + q15_t inV = *pA++; + q7_t inM = *pB++; + sum += inV * inM; + colCnt--; + } + + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + + rowCnt--; + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + uint16_t rowCnt = num_of_rows >> 2; + const q7_t *pB = pM; + const q15_t *pA; + q15_t *pO = pOut; + const q7_t *pBias = bias; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + uint16_t colCnt = dim_vec >> 1; + + pA = pV; + + while (colCnt) + { + q15_t inA1 = *pA++; + q15_t inA2 = *pA++; + + q7_t inB1 = *pB++; + q7_t inB3 = *pB++; + q7_t inB2 = *pB++; + q7_t inB4 = *pB++; + + sum += inA1 * inB1 + inA2 * inB2; + sum2 += inA1 * inB3 + inA2 * inB4; + + inB1 = *pB++; + inB3 = *pB++; + inB2 = *pB++; + inB4 = *pB++; + + sum3 += inA1 * inB1 + inA2 * inB2; + sum4 += inA1 * inB3 + inA2 * inB4; + + colCnt--; + } + + colCnt = dim_vec & 0x1; + while (colCnt) + { + q15_t inA = *pA++; + q7_t inB = *pB++; + sum += inA * inB; + inB = *pB++; + sum2 += inA * inB; + inB = *pB++; + sum3 += inA * inB; + inB = *pB++; + sum4 += inA * inB; + + colCnt--; + } + *pO++ = (q15_t) __SSAT((sum >> out_shift), 16); + *pO++ = (q15_t) __SSAT((sum2 >> out_shift), 16); + *pO++ = (q15_t) __SSAT((sum3 >> out_shift), 16); + *pO++ = (q15_t) __SSAT((sum4 >> out_shift), 16); + + rowCnt--; + } + + rowCnt = num_of_rows & 0x3; + + while (rowCnt) + { + int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + int j; + + pA = pV; + for (j = 0; j < dim_vec; j++) + { + q15_t inA = *pA++; + q7_t inB = *pB++; + ip_out += inA * inB; + } + *pO++ = (q15_t) __SSAT((ip_out >> out_shift), 16); + + rowCnt--; + } + +#endif /* ARM_MATH_DSP */ + + /* Return to ARM_MATH_SUCCESS */ + return (ARM_MATH_SUCCESS); + +} + +/** + * @} end of FC group + */ diff --git a/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c new file mode 100644 index 0000000..c3e7cf2 --- /dev/null +++ b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_fully_connected_q15.c + * Description: Q15 basic fully-connected layer function + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup FC + * @{ + */ + + /** + * @brief Q15 opt fully-connected layer function + * @param[in] pV pointer to input vector + * @param[in] pM pointer to matrix weights + * @param[in] dim_vec length of the vector + * @param[in] num_of_rows number of rows in weight matrix + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias pointer to bias + * @param[in,out] pOut pointer to output vector + * @param[in,out] vec_buffer pointer to buffer space for input + * @return The function returns ARM_MATH_SUCCESS + * + * + * @details + * + * Buffer size: + * + * vec_buffer size: 0 + * + */ + +arm_status +arm_fully_connected_q15(const q15_t * pV, + const q15_t * pM, + const uint16_t dim_vec, + const uint16_t num_of_rows, + const uint16_t bias_shift, + const uint16_t out_shift, + const q15_t * bias, + q15_t * pOut, + q15_t * vec_buffer) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + const q15_t *pB = pM; + const q15_t *pB2 = pB + dim_vec; + q15_t *pO = pOut; + const q15_t *pA; + const q15_t *pBias = bias; + uint16_t rowCnt = num_of_rows >> 1; + + /* this loop loops over different output */ + while (rowCnt) { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 2; + + pA = pV; + pB2 = pB + dim_vec; + + while (colCnt) + { + q31_t inV1, inM1, inM2; + inV1 = *__SIMD32(pA)++; + inM1 = *__SIMD32(pB)++; + sum = __SMLAD(inV1, inM1, sum); + inM2 = *__SIMD32(pB2)++; + sum2 = __SMLAD(inV1, inM2, sum2); + + inV1 = *__SIMD32(pA)++; + inM1 = *__SIMD32(pB)++; + sum = __SMLAD(inV1, inM1, sum); + inM2 = *__SIMD32(pB2)++; + sum2 = __SMLAD(inV1, inM2, sum2); + + colCnt--; + } + colCnt = dim_vec & 0x3; + while (colCnt) + { + q15_t inV = *pA++; + q15_t inM = *pB++; + q15_t inM2 = *pB2++; + + sum += inV * inM; + sum2 += inV * inM2; + colCnt--; + } /* while over colCnt */ + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum2>> out_shift), 16)); + + /* adjust the pointers and counters */ + pB = pB + dim_vec; + rowCnt --; + } + + rowCnt = num_of_rows & 0x1; + + while (rowCnt) { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 2; + + pA = pV; + + while (colCnt) { + q31_t inV1, inM1; + inV1 = *__SIMD32(pA)++; + inM1 = *__SIMD32(pB)++; + sum = __SMLAD(inV1, inM1, sum); + + inV1 = *__SIMD32(pA)++; + inM1 = *__SIMD32(pB)++; + sum = __SMLAD(inV1, inM1, sum); + + colCnt--; + } + + /* left-over of the vector */ + colCnt = dim_vec & 0x3; + while(colCnt) { + q15_t inV = *pA++; + q15_t inM = *pB++; + + sum += inV * inM; + + colCnt--; + } + + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + + rowCnt --; + } + +#else + int i, j; + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + for (i = 0; i < num_of_rows; i++) + { + int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + for (j = 0; j < dim_vec; j++) + { + ip_out += pV[j] * pM[i * dim_vec + j]; + } + pOut[i] = (q15_t) __SSAT((ip_out >> out_shift), 16); + } + +#endif /* ARM_MATH_DSP */ + + /* Return to application */ + return (ARM_MATH_SUCCESS); + +} + +/** + * @} end of FC group + */ diff --git a/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c new file mode 100644 index 0000000..f7a3915 --- /dev/null +++ b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c @@ -0,0 +1,332 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_fully_connected_q15_opt.c + * Description: Q15 opt fully-connected layer function + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup FC + * @{ + */ + + /** + * @brief Q15 opt fully-connected layer function + * @param[in] pV pointer to input vector + * @param[in] pM pointer to matrix weights + * @param[in] dim_vec length of the vector + * @param[in] num_of_rows number of rows in weight matrix + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias pointer to bias + * @param[in,out] pOut pointer to output vector + * @param[in,out] vec_buffer pointer to buffer space for input + * @return The function returns ARM_MATH_SUCCESS + * + * + * @details + * + * Buffer size: + * + * vec_buffer size: 0 + * + * Here we use only one pointer to read 4 rows in the weight + * matrix. So if the original matrix looks like this: + * + * | a11 | a12 | a13 | + * + * | a21 | a22 | a23 | + * + * | a31 | a32 | a33 | + * + * | a41 | a42 | a43 | + * + * | a51 | a52 | a53 | + * + * | a61 | a62 | a63 | + * + * We operates on multiple-of-4 rows, so the first four rows becomes + * + * | a11 | a12 | a21 | a22 | a31 | a32 | a41 | a42 | + * + * | a13 | a23 | a33 | a43 | + * + * Remaining rows are kept the same original order. + * + * So the stored weight matrix looks like this: + * + * + * | a11 | a12 | a21 | a22 | a31 | a32 | a41 | a42 | + * + * | a13 | a23 | a33 | a43 | a51 | a52 | a53 | a61 | + * + * | a62 | a63 | + */ + +arm_status +arm_fully_connected_q15_opt(const q15_t * pV, + const q15_t * pM, + const uint16_t dim_vec, + const uint16_t num_of_rows, + const uint16_t bias_shift, + const uint16_t out_shift, + const q15_t * bias, + q15_t * pOut, + q15_t * vec_buffer) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + const q15_t *pB = pM; + q15_t *pO = pOut; + const q15_t *pBias = bias; + const q15_t *pA = pV; + + uint16_t rowCnt = num_of_rows >> 2; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 1; + + pA = pV; + +#ifdef USE_INTRINSIC + + while (colCnt) + { + q31_t inM11, inM12, inM13, inM14; + q31_t inV; + + inV = *__SIMD32(pA)++; + inM11 = *__SIMD32(pB)++; + sum = __SMLAD(inV, inM11, sum); + inM12 = *__SIMD32(pB)++; + sum2 = __SMLAD(inV, inM12, sum2); + inM13 = *__SIMD32(pB)++; + sum3 = __SMLAD(inV, inM13, sum3); + inM14 = *__SIMD32(pB)++; + sum4 = __SMLAD(inV, inM14, sum4); + colCnt--; + } + +#else + + /* + * register needed: + * loop counter: colCnt + * accumulators: sum, sum2, sum3, sum4 + * pointers: pB, pA + * weight data: inM11, inM12, inM13, inM14 + * activation data: inV + */ + + asm volatile ("COL_LOOP_%=:\n" + "ldr.w r4, [%[pA]], #4\n" + "ldr.w r0, [%[pB]], #16\n" + "smlad %[sum], r4, r0, %[sum]\n" + "ldr.w r1, [%[pB] , #-12]\n" + "smlad %[sum2], r4, r1, %[sum2]\n" + "ldr.w r2, [%[pB] , #-8]\n" + "smlad %[sum3], r4, r2, %[sum3]\n" + "ldr.w r3, [%[pB] , #-4]\n" + "smlad %[sum4], r4, r3, %[sum4]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP_%=\n":[sum] "+r"(sum), + [sum2] "+r"(sum2),[sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4"); + +#endif /* USE_INTRINSIC */ + + colCnt = dim_vec & 0x1; + while (colCnt) + { + + q15_t inV = *pA++; + q15_t inM = *pB++; + q15_t inM2 = *pB++; + q15_t inM3 = *pB++; + q15_t inM4 = *pB++; + + sum += inV * inM; + sum2 += inV * inM2; + sum3 += inV * inM3; + sum4 += inV * inM4; + colCnt--; + } /* while over colCnt */ + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum2 >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum3 >> out_shift), 16)); + *pO++ = (q15_t) (__SSAT((sum4 >> out_shift), 16)); + + /* adjust the pointers and counters */ + rowCnt--; + } + + /* left-over part of the rows */ + rowCnt = num_of_rows & 0x3; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 2; + + pA = pV; + + while (colCnt) + { + q31_t inV1, inV2, inM1, inM2; + + inM1 = *__SIMD32(pB)++; + inV1 = *__SIMD32(pA)++; + sum = __SMLAD(inV1, inM1, sum); + + inM2 = *__SIMD32(pB)++; + inV2 = *__SIMD32(pA)++; + sum = __SMLAD(inV2, inM2, sum); + + colCnt--; + } + + /* left-over of the vector */ + colCnt = dim_vec & 0x3; + while (colCnt) + { + q15_t inV = *pA++; + q15_t inM = *pB++; + sum += inV * inM; + colCnt--; + } + + *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16)); + + rowCnt--; + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + uint16_t rowCnt = num_of_rows >> 2; + const q15_t *pB = pM; + const q15_t *pA; + q15_t *pO = pOut; + const q15_t *pBias = bias; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 1; + + pA = pV; + while (colCnt) + { + q15_t inA1 = *pA++; + q15_t inA2 = *pA++; + + q15_t inB1 = *pB++; + q15_t inB2 = *pB++; + sum += inA1 * inB1 + inA2 * inB2; + + inB1 = *pB++; + inB2 = *pB++; + sum2 += inA1 * inB1 + inA2 * inB2; + + inB1 = *pB++; + inB2 = *pB++; + sum3 += inA1 * inB1 + inA2 * inB2; + + inB1 = *pB++; + inB2 = *pB++; + sum4 += inA1 * inB1 + inA2 * inB2; + + colCnt--; + } + colCnt = dim_vec & 0x1; + while (colCnt) + { + q15_t inA = *pA++; + q15_t inB = *pB++; + sum += inA * inB; + inB = *pB++; + sum2 += inA * inB; + inB = *pB++; + sum3 += inA * inB; + inB = *pB++; + sum4 += inA * inB; + colCnt--; + } + *pO++ = (q15_t) __SSAT((sum >> out_shift), 16); + *pO++ = (q15_t) __SSAT((sum2 >> out_shift), 16); + *pO++ = (q15_t) __SSAT((sum3 >> out_shift), 16); + *pO++ = (q15_t) __SSAT((sum4 >> out_shift), 16); + + rowCnt--; + } + rowCnt = num_of_rows & 0x3; + + while (rowCnt) + { + int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + int j; + + pA = pV; + for (j = 0; j < dim_vec; j++) + { + q15_t inA = *pA++; + q15_t inB = *pB++; + ip_out += inA * inB; + } + *pO++ = (q15_t) __SSAT((ip_out >> out_shift), 16); + + rowCnt--; + } + +#endif /* ARM_MATH_DSP */ + + /* Return to ARM_MATH_SUCCESS */ + return (ARM_MATH_SUCCESS); + +} + +/** + * @} end of FC group + */ diff --git a/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c new file mode 100644 index 0000000..d8efc04 --- /dev/null +++ b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c @@ -0,0 +1,198 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_fully_connected_q7.c + * Description: Q7 basic fully-connected layer function + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup FC + * @{ + */ + + /** + * @brief Q7 basic fully-connected layer function + * @param[in] pV pointer to input vector + * @param[in] pM pointer to matrix weights + * @param[in] dim_vec length of the vector + * @param[in] num_of_rows number of rows in weight matrix + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias pointer to bias + * @param[in,out] pOut pointer to output vector + * @param[in,out] vec_buffer pointer to buffer space for input + * @return The function returns ARM_MATH_SUCCESS + * + * @details + * + * Buffer size: + * + * vec_buffer size: dim_vec + * + * This basic function is designed to work with regular weight + * matrix without interleaving. + * + */ + +arm_status +arm_fully_connected_q7(const q7_t * pV, + const q7_t * pM, + const uint16_t dim_vec, + const uint16_t num_of_rows, + const uint16_t bias_shift, + const uint16_t out_shift, const q7_t * bias, q7_t * pOut, q15_t * vec_buffer) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + const q7_t *pB = pM; + const q7_t *pB2; + q7_t *pO = pOut; + const q7_t *pBias = bias; + q15_t *pA; + uint16_t rowCnt = num_of_rows >> 1; + + /* expand the vector into the buffer */ + arm_q7_to_q15_reordered_no_shift(pV, vec_buffer, dim_vec); + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + uint16_t colCnt = dim_vec >> 2; + + pA = vec_buffer; + pB2 = pB + dim_vec; + + while (colCnt) + { + q31_t inV, inM11, inM12, inM21, inM22; + pB = (q7_t *) read_and_pad_reordered((void *)pB, &inM11, &inM12); + pB2 = (q7_t *) read_and_pad_reordered((void *)pB2, &inM21, &inM22); + + inV = *__SIMD32(pA)++; + + sum = __SMLAD(inV, inM11, sum); + sum2 = __SMLAD(inV, inM21, sum2); + + inV = *__SIMD32(pA)++; + + sum = __SMLAD(inV, inM12, sum); + sum2 = __SMLAD(inV, inM22, sum2); + + colCnt--; + } + colCnt = dim_vec & 0x3; + while (colCnt) + { + q7_t inV = *pA++; + q15_t inM = *pB++; + q15_t inM2 = *pB2++; + + sum += inV * inM; + sum2 += inV * inM2; + colCnt--; + } /* while over colCnt */ + *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8)); + *pO++ = (q7_t) (__SSAT((sum2 >> out_shift), 8)); + + /* adjust the pointers and counters */ + pB += dim_vec; + rowCnt--; + } + + /* left-over part of the rows */ + rowCnt = num_of_rows & 0x1; + + while (rowCnt) + { + uint16_t colCnt = dim_vec >> 2; + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + pA = vec_buffer; + + while (colCnt) + { + q31_t inV1, inV2, inM11, inM12; + + pB = (q7_t *) read_and_pad_reordered((void *)pB, &inM11, &inM12); + + inV1 = *__SIMD32(pA)++; + sum = __SMLAD(inV1, inM11, sum); + + inV2 = *__SIMD32(pA)++; + sum = __SMLAD(inV2, inM12, sum); + + colCnt--; + } + + /* left-over of the vector */ + colCnt = dim_vec & 0x3; + while (colCnt) + { + q7_t inV = *pA++; + q15_t inM = *pB++; + sum += inV * inM; + colCnt--; + } + + *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8)); + + rowCnt--; + } + +#else + int i, j; + + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + for (i = 0; i < num_of_rows; i++) + { + int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); + for (j = 0; j < dim_vec; j++) + { + ip_out += pV[j] * pM[i * dim_vec + j]; + } + pOut[i] = (q7_t) __SSAT((ip_out >> out_shift), 8); + } + +#endif /* ARM_MATH_DSP */ + + /* Return to ARM_MATH_SUCCESS */ + return (ARM_MATH_SUCCESS); + +} + +/** + * @} end of FC group + */ diff --git a/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c new file mode 100644 index 0000000..e3d0874 --- /dev/null +++ b/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c @@ -0,0 +1,484 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_fully_connected_q7_opt.c + * Description: Q7 basic fully-connected layer function + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup FC + * @{ + */ + + /** + * @brief Q7 opt fully-connected layer function + * @param[in] pV pointer to input vector + * @param[in] pM pointer to matrix weights + * @param[in] dim_vec length of the vector + * @param[in] num_of_rows number of rows in weight matrix + * @param[in] bias_shift amount of left-shift for bias + * @param[in] out_shift amount of right-shift for output + * @param[in] bias pointer to bias + * @param[in,out] pOut pointer to output vector + * @param[in,out] vec_buffer pointer to buffer space for input + * @return The function returns ARM_MATH_SUCCESS + * + * @details + * + * Buffer size: + * + * vec_buffer size: dim_vec + * + * This opt function is designed to work with interleaved weight + * matrix. The vector input is assumed in q7_t format, we call + * arm_q7_to_q15_no_shift_shuffle function to expand into + * q15_t format with certain weight re-ordering, refer to the function + * comments for more details. + * Here we use only one pointer to read 4 rows in the weight + * matrix. So if the original q7_t matrix looks like this: + * + * | a11 | a12 | a13 | a14 | a15 | a16 | a17 | + * + * | a21 | a22 | a23 | a24 | a25 | a26 | a27 | + * + * | a31 | a32 | a33 | a34 | a35 | a36 | a37 | + * + * | a41 | a42 | a43 | a44 | a45 | a46 | a47 | + * + * | a51 | a52 | a53 | a54 | a55 | a56 | a57 | + * + * | a61 | a62 | a63 | a64 | a65 | a66 | a67 | + * + * + * We operates on multiple-of-4 rows, so the first four rows becomes + * + * | a11 | a21 | a13 | a23 | a31 | a41 | a33 | a43 | + * + * | a12 | a22 | a14 | a24 | a32 | a42 | a34 | a44 | + * + * | a15 | a25 | a35 | a45 | a16 | a26 | a36 | a46 | + * + * So within the kernel, we first read the re-ordered vector in as: + * + * | b1 | b3 | and | b2 | b4 | + * + * the four q31_t weights will look like + * + * | a11 | a13 |, | a21 | a23 |, | a31 | a33 |, | a41 | a43 | + * + * | a12 | a14 |, | a22 | a24 |, | a32 | a34 |, | a42 | a44 | + * + * The column left over will be in-order. + * which is: + * + * | a17 | a27 | a37 | a47 | + * + * For the left-over rows, we do 1x1 computation, so the data remains + * as its original order. + * + * So the stored weight matrix looks like this: + * + * | a11 | a21 | a13 | a23 | a31 | a41 | + * + * | a33 | a43 | a12 | a22 | a14 | a24 | + * + * | a32 | a42 | a34 | a44 | a15 | a25 | + * + * | a35 | a45 | a16 | a26 | a36 | a46 | + * + * | a17 | a27 | a37 | a47 | a51 | a52 | + * + * | a53 | a54 | a55 | a56 | a57 | a61 | + * + * | a62 | a63 | a64 | a65 | a66 | a67 | + * + * + */ + +arm_status +arm_fully_connected_q7_opt(const q7_t * pV, + const q7_t * pM, + const uint16_t dim_vec, + const uint16_t num_of_rows, + const uint16_t bias_shift, + const uint16_t out_shift, + const q7_t * bias, + q7_t * pOut, + q15_t * vec_buffer) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + const q7_t *pB = pM; + q7_t *pO = pOut; + const q7_t *pBias = bias; + q15_t *pA; + uint16_t rowCnt = num_of_rows >> 2; + + arm_q7_to_q15_reordered_no_shift(pV, vec_buffer, dim_vec); + + while (rowCnt) + { + + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 2; + + pA = vec_buffer; + +#ifdef USE_INTRINSIC + +#ifndef ARM_MATH_BIG_ENDIAN + while (colCnt) + { + q31_t inM11, inM12, inM13, inM14; + q31_t inV; + + inV = *__SIMD32(pA)++; + inM11 = *__SIMD32(pB)++; + inM12 = __SXTB16(__ROR(inM11, 8)); + inM11 = __SXTB16(inM11); + sum = __SMLAD(inM11, inV, sum); + sum2 = __SMLAD(inM12, inV, sum2); + inM13 = *__SIMD32(pB)++; + inM14 = __SXTB16(__ROR(inM13, 8)); + inM13 = __SXTB16(inM13); + sum3 = __SMLAD(inM13, inV, sum3); + sum4 = __SMLAD(inM14, inV, sum4); + + inV = *__SIMD32(pA)++; + inM11 = *__SIMD32(pB)++; + inM12 = __SXTB16(__ROR(inM11, 8)); + inM11 = __SXTB16(inM11); + sum = __SMLAD(inM11, inV, sum); + sum2 = __SMLAD(inM12, inV, sum2); + inM13 = *__SIMD32(pB)++; + inM14 = __SXTB16(__ROR(inM13, 8)); + inM13 = __SXTB16(inM13); + sum3 = __SMLAD(inM13, inV, sum3); + sum4 = __SMLAD(inM14, inV, sum4); + colCnt--; + } +#else + while (colCnt) + { + q31_t inM11, inM12, inM13, inM14; + q31_t inV; + + inV = *__SIMD32(pA)++; + inM11 = *__SIMD32(pB)++; + inM12 = __SXTB16(__ROR(inM11, 8)); + inM11 = __SXTB16(inM11); + sum = __SMLAD(inM12, inV, sum); + sum2 = __SMLAD(inM11, inV, sum2); + inM13 = *__SIMD32(pB)++; + inM14 = __SXTB16(__ROR(inM13, 8)); + inM13 = __SXTB16(inM13); + sum3 = __SMLAD(inM14, inV, sum3); + sum4 = __SMLAD(inM13, inV, sum4); + + inV = *__SIMD32(pA)++; + inM11 = *__SIMD32(pB)++; + inM12 = __SXTB16(__ROR(inM11, 8)); + inM11 = __SXTB16(inM11); + sum = __SMLAD(inM12, inV, sum); + sum2 = __SMLAD(inM11, inV, sum2); + inM13 = *__SIMD32(pB)++; + inM14 = __SXTB16(__ROR(inM13, 8)); + inM13 = __SXTB16(inM13); + sum3 = __SMLAD(inM14, inV, sum3); + sum4 = __SMLAD(inM13, inV, sum4); + colCnt--; + } +#endif /* ARM_MATH_BIG_ENDIAN */ + +#else + + /* + * register needed: + * loop counter: colCnt + * accumulators: sum, sum2, sum3, sum4 + * pointers: pB, pA + * weight data: inM11, inM12, inM13, inM14 + * activation data: inV + */ + +#ifndef ARM_MATH_BIG_ENDIAN + asm volatile ("COL_LOOP_%=:\n" + "ldr.w r4, [%[pA]], #8\n" + "ldr.w r1, [%[pB]], #16\n" + "mov.w r0, r1, ror #8\n" + "sxtb16 r0, r0\n" + "sxtb16 r1, r1\n" + "smlad %[sum], r4, r1, %[sum]\n" + "smlad %[sum2], r4, r0, %[sum2]\n" + "ldr.w r3, [%[pB], #-12]\n" + "mov.w r2, r3, ror #8\n" + "sxtb16 r2, r2\n" + "sxtb16 r3, r3\n" + "smlad %[sum3], r4, r3, %[sum3]\n" + "smlad %[sum4], r4, r2, %[sum4]\n" + "ldr.w r4, [%[pA], #-4]\n" + "ldr.w r1, [%[pB], #-8]\n" + "mov.w r0, r1, ror #8\n" + "sxtb16 r0, r0\n" + "sxtb16 r1, r1\n" + "smlad %[sum], r4, r1, %[sum]\n" + "smlad %[sum2], r4, r0, %[sum2]\n" + "ldr.w r3, [%[pB], #-4]\n" + "mov.w r2, r3, ror #8\n" + "sxtb16 r2, r2\n" + "sxtb16 r3, r3\n" + "smlad %[sum3], r4, r3, %[sum3]\n" + "smlad %[sum4], r4, r2, %[sum4]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP_%=\n":[sum] "+r"(sum), + [sum2] "+r"(sum2),[sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4"); +#else + asm volatile ("COL_LOOP_%=:\n" + "ldr.w r4, [%[pA]], #8\n" + "ldr.w r1, [%[pB]], #16\n" + "mov.w r0, r1, ror #8\n" + "sxtb16 r0, r0\n" + "sxtb16 r1, r1\n" + "smlad %[sum], r4, r0, %[sum]\n" + "smlad %[sum2], r4, r1, %[sum2]\n" + "ldr.w r3, [%[pB], #-12]\n" + "mov.w r2, r3, ror #8\n" + "sxtb16 r2, r2\n" + "sxtb16 r3, r3\n" + "smlad %[sum3], r4, r2, %[sum3]\n" + "smlad %[sum4], r4, r3, %[sum4]\n" + "ldr.w r4, [%[pA], #-4]\n" + "ldr.w r1, [%[pB], #-8]\n" + "mov.w r0, r1, ror #8\n" + "sxtb16 r0, r0\n" + "sxtb16 r1, r1\n" + "smlad %[sum], r4, r0, %[sum]\n" + "smlad %[sum2], r4, r1, %[sum2]\n" + "ldr.w r3, [%[pB], #-4]\n" + "mov.w r2, r3, ror #8\n" + "sxtb16 r2, r2\n" + "sxtb16 r3, r3\n" + "smlad %[sum3], r4, r2, %[sum3]\n" + "smlad %[sum4], r4, r3, %[sum4]\n" + "subs %[colCnt], #1\n" + "bne COL_LOOP_%=\n":[sum] "+r"(sum), + [sum2] "+r"(sum2),[sum3] "+r"(sum3), + [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4"); +#endif /* ARM_MATH_BIG_ENDIAN */ + +#endif /* USE_INTRINSIC */ + + colCnt = dim_vec & 0x3; + while (colCnt) + { + q15_t inV = *pA++; + q7_t inM = *pB++; + q7_t inM2 = *pB++; + q7_t inM3 = *pB++; + q7_t inM4 = *pB++; + + sum += inV * inM; + sum2 += inV * inM2; + sum3 += inV * inM3; + sum4 += inV * inM4; + colCnt--; + } /* while over colCnt */ + *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8)); + *pO++ = (q7_t) (__SSAT((sum2 >> out_shift), 8)); + *pO++ = (q7_t) (__SSAT((sum3 >> out_shift), 8)); + *pO++ = (q7_t) (__SSAT((sum4 >> out_shift), 8)); + + /* adjust the pointers and counters */ + rowCnt--; + } + + /* left-over part of the rows */ + rowCnt = num_of_rows & 0x3; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + uint16_t colCnt = dim_vec >> 2; + + pA = vec_buffer; + + while (colCnt) + { + q31_t inV1, inV2, inM11, inM12; + + pB = (q7_t *) read_and_pad_reordered((void *)pB, &inM11, &inM12); + + inV1 = *__SIMD32(pA)++; + sum = __SMLAD(inV1, inM11, sum); + + inV2 = *__SIMD32(pA)++; + sum = __SMLAD(inV2, inM12, sum); + + colCnt--; + } + + /* left-over of the vector */ + colCnt = dim_vec & 0x3; + while (colCnt) + { + q15_t inV = *pA++; + q7_t inM = *pB++; + sum += inV * inM; + colCnt--; + } + + *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8)); + + rowCnt--; + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + uint16_t rowCnt = num_of_rows >> 2; + const q7_t *pB = pM; + const q7_t *pA; + q7_t *pO = pOut; + const q7_t *pBias = bias; + + while (rowCnt) + { + q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + uint16_t colCnt = dim_vec >> 2; + + pA = pV; + + while (colCnt) + { + q7_t inA1 = *pA++; + q7_t inA3 = *pA++; + q7_t inA2 = *pA++; + q7_t inA4 = *pA++; + + q7_t inB1 = *pB++; + q7_t inB3 = *pB++; + q7_t inB2 = *pB++; + q7_t inB4 = *pB++; + + sum += inA1 * inB1 + inA2 * inB2; + sum2 += inA1 * inB3 + inA2 * inB4; + + inB1 = *pB++; + inB3 = *pB++; + inB2 = *pB++; + inB4 = *pB++; + + sum3 += inA1 * inB1 + inA2 * inB2; + sum4 += inA1 * inB3 + inA2 * inB4; + + inB1 = *pB++; + inB3 = *pB++; + inB2 = *pB++; + inB4 = *pB++; + + sum += inA3 * inB1 + inA4 * inB2; + sum2 += inA3 * inB3 + inA4 * inB4; + + inB1 = *pB++; + inB3 = *pB++; + inB2 = *pB++; + inB4 = *pB++; + + sum3 += inA3 * inB1 + inA4 * inB2; + sum4 += inA3 * inB3 + inA4 * inB4; + + colCnt--; + } + colCnt = dim_vec & 0x3; + while (colCnt) + { + q7_t inA = *pA++; + q7_t inB = *pB++; + sum += inA * inB; + inB = *pB++; + sum2 += inA * inB; + inB = *pB++; + sum3 += inA * inB; + inB = *pB++; + sum4 += inA * inB; + + colCnt--; + } + *pO++ = (q7_t) __SSAT((sum >> out_shift), 8); + *pO++ = (q7_t) __SSAT((sum2 >> out_shift), 8); + *pO++ = (q7_t) __SSAT((sum3 >> out_shift), 8); + *pO++ = (q7_t) __SSAT((sum4 >> out_shift), 8); + + rowCnt--; + } + + rowCnt = num_of_rows & 0x3; + + while (rowCnt) + { + int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); + + int j; + + pA = pV; + for (j = 0; j < dim_vec; j++) + { + q7_t inA = *pA++; + q7_t inB = *pB++; + ip_out += inA * inB; + } + *pO++ = (q7_t) __SSAT((ip_out >> out_shift), 8); + + rowCnt--; + } + +#endif /* ARM_MATH_DSP */ + + /* Return to ARM_MATH_SUCCESS */ + return (ARM_MATH_SUCCESS); + +} + +/** + * @} end of FC group + */ diff --git a/NN/Source/NNSupportFunctions/arm_nn_mult_q15.c b/NN/Source/NNSupportFunctions/arm_nn_mult_q15.c new file mode 100644 index 0000000..5a60459 --- /dev/null +++ b/NN/Source/NNSupportFunctions/arm_nn_mult_q15.c @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_nn_mult_q15.c + * Description: Q15 vector multiplication with variable output shifts + * + * $Date: 13. July 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_nnfunctions.h" + +/** + * @ingroup groupSupport + */ + +/** + * @addtogroup NNBasicMath + * @{ + */ + + +/** + * @brief Q7 vector multiplication with variable output shifts + * @param[in] *pSrcA pointer to the first input vector + * @param[in] *pSrcB pointer to the second input vector + * @param[out] *pDst pointer to the output vector + * @param[in] out_shift amount of right-shift for output + * @param[in] blockSize number of samples in each vector + * @return none. + * + * Scaling and Overflow Behavior: + * \par + * The function uses saturating arithmetic. + * Results outside of the allowable Q15 range [0x8000 0x7FFF] will be saturated. + */ + +void arm_nn_mult_q15( + q15_t * pSrcA, + q15_t * pSrcB, + q15_t * pDst, + const uint16_t out_shift, + uint32_t blockSize) +{ + uint32_t blkCnt; /* loop counters */ + +#if defined (ARM_MATH_DSP) + +/* Run the below code for Cortex-M4 and Cortex-M3 */ + q31_t inA1, inA2, inB1, inB2; /* temporary input variables */ + q15_t out1, out2, out3, out4; /* temporary output variables */ + q31_t mul1, mul2, mul3, mul4; /* temporary variables */ + + /* loop Unrolling */ + blkCnt = blockSize >> 2U; + + /* First part of the processing with loop unrolling. Compute 4 outputs at a time. + ** a second loop below computes the remaining 1 to 3 samples. */ + while (blkCnt > 0U) + { + /* read two samples at a time from sourceA */ + inA1 = *__SIMD32(pSrcA)++; + /* read two samples at a time from sourceB */ + inB1 = *__SIMD32(pSrcB)++; + /* read two samples at a time from sourceA */ + inA2 = *__SIMD32(pSrcA)++; + /* read two samples at a time from sourceB */ + inB2 = *__SIMD32(pSrcB)++; + + /* multiply mul = sourceA * sourceB */ + mul1 = (q31_t) ((q15_t) (inA1 >> 16) * (q15_t) (inB1 >> 16)); + mul2 = (q31_t) ((q15_t) inA1 * (q15_t) inB1); + mul3 = (q31_t) ((q15_t) (inA2 >> 16) * (q15_t) (inB2 >> 16)); + mul4 = (q31_t) ((q15_t) inA2 * (q15_t) inB2); + + /* saturate result to 16 bit */ + out1 = (q15_t) __SSAT((mul1 + NN_ROUND(out_shift)) >> out_shift, 16); + out2 = (q15_t) __SSAT((mul2 + NN_ROUND(out_shift)) >> out_shift, 16); + out3 = (q15_t) __SSAT((mul3 + NN_ROUND(out_shift)) >> out_shift, 16); + out4 = (q15_t) __SSAT((mul4 + NN_ROUND(out_shift)) >> out_shift, 16); + + /* store the result */ +#ifndef ARM_MATH_BIG_ENDIAN + + *__SIMD32(pDst)++ = __PKHBT(out2, out1, 16); + *__SIMD32(pDst)++ = __PKHBT(out4, out3, 16); + +#else + + *__SIMD32(pDst)++ = __PKHBT(out2, out1, 16); + *__SIMD32(pDst)++ = __PKHBT(out4, out3, 16); + +#endif /* #ifndef ARM_MATH_BIG_ENDIAN */ + + /* Decrement the blockSize loop counter */ + blkCnt--; + } + + /* If the blockSize is not a multiple of 4, compute any remaining output samples here. + ** No loop unrolling is used. */ + blkCnt = blockSize % 0x4U; + +#else + + /* Run the below code for Cortex-M0 */ + + /* Initialize blkCnt with number of samples */ + blkCnt = blockSize; + +#endif /* #if defined (ARM_MATH_DSP) */ + + + while (blkCnt > 0U) + { + /* C = A * B */ + /* Multiply the inputs and store the result in the destination buffer */ + *pDst++ = (q15_t) __SSAT((((q31_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 16); + + /* Decrement the blockSize loop counter */ + blkCnt--; + } +} + +/** + * @} end of NNBasicMath group + */ + diff --git a/NN/Source/NNSupportFunctions/arm_nn_mult_q7.c b/NN/Source/NNSupportFunctions/arm_nn_mult_q7.c new file mode 100644 index 0000000..3735c04 --- /dev/null +++ b/NN/Source/NNSupportFunctions/arm_nn_mult_q7.c @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_nn_mult_q7.c + * Description: Q7 vector multiplication with variable output shifts + * + * $Date: 13. July 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_nnfunctions.h" + +/** + * @ingroup groupSupport + */ + +/** + * @addtogroup NNBasicMath + * @{ + */ + +/** + * @brief Q7 vector multiplication with variable output shifts + * @param[in] *pSrcA pointer to the first input vector + * @param[in] *pSrcB pointer to the second input vector + * @param[out] *pDst pointer to the output vector + * @param[in] out_shift amount of right-shift for output + * @param[in] blockSize number of samples in each vector + * @return none. + * + * Scaling and Overflow Behavior: + * \par + * The function uses saturating arithmetic. + * Results outside of the allowable Q7 range [0x80 0x7F] will be saturated. + */ + +void arm_nn_mult_q7( + q7_t * pSrcA, + q7_t * pSrcB, + q7_t * pDst, + const uint16_t out_shift, + uint32_t blockSize) +{ + uint32_t blkCnt; /* loop counters */ + +#if defined (ARM_MATH_DSP) + +/* Run the below code for Cortex-M4 and Cortex-M3 */ + q7_t out1, out2, out3, out4; /* Temporary variables to store the product */ + + /* loop Unrolling */ + blkCnt = blockSize >> 2U; + + /* First part of the processing with loop unrolling. Compute 4 outputs at a time. + ** a second loop below computes the remaining 1 to 3 samples. */ + while (blkCnt > 0U) + { + /* C = A * B */ + /* Multiply the inputs and store the results in temporary variables */ + out1 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8); + out2 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8); + out3 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8); + out4 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8); + + /* Store the results of 4 inputs in the destination buffer in single cycle by packing */ + *__SIMD32(pDst)++ = __PACKq7(out1, out2, out3, out4); + + /* Decrement the blockSize loop counter */ + blkCnt--; + } + + /* If the blockSize is not a multiple of 4, compute any remaining output samples here. + ** No loop unrolling is used. */ + blkCnt = blockSize % 0x4U; + +#else + + /* Run the below code for Cortex-M0 */ + + /* Initialize blkCnt with number of samples */ + blkCnt = blockSize; + +#endif /* #if defined (ARM_MATH_DSP) */ + + + while (blkCnt > 0U) + { + /* C = A * B */ + /* Multiply the inputs and store the result in the destination buffer */ + *pDst++ = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8); + + /* Decrement the blockSize loop counter */ + blkCnt--; + } +} + +/** + * @} end of NNBasicMath group + */ diff --git a/NN/Source/NNSupportFunctions/arm_nntables.c b/NN/Source/NNSupportFunctions/arm_nntables.c new file mode 100644 index 0000000..c28f1a6 --- /dev/null +++ b/NN/Source/NNSupportFunctions/arm_nntables.c @@ -0,0 +1,297 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_nntables.c + * Description: Converts the elements of the Q7 vector to Q15 vector without left-shift + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_nnsupportfunctions.h" + +/** + * @brief tables for various activation functions + * + * This file include the declaration of common tables. + * Most of them are used for activation functions + * + * Assumption: + * Unified table: input is 3.x format, i.e, range of [-8, 8) + * sigmoid(8) = 0.9996646498695336 + * tanh(8) = 0.9999997749296758 + * The accuracy here should be good enough + * + * 2-stage HL table: + * + * The entire input range is divided into two parts: + * + * Low range table: 0x000x xxxx or 0x111x xxxx + * table entry will be the binary number excluding the first + * two digits, i.e., 0x0x xxxx or 0x1x xxxx + * + * + * + * High range table 0x0010 0000 -- 0x0111 1111 + * 0x1000 0000 -- 0x1101 1111 + * + * For positive numbers, table entry will be + * 0x0010 0000 -- 0x0111 1111 minus 0x0010 0000 + * i.e., 0x0000 0000 - 0x0101 11111 + * + * same thing for the negative numbers, table entry will be + * 0x1000 0000 -- 0x1101 1111 minux 0x0010 0000 + * i.e., 0x0110 0000 - 0x1011 1111 + */ + +const q7_t sigmoidTable_q7[256] = { + 0x40, 0x42, 0x44, 0x46, 0x48, 0x4a, 0x4c, 0x4e, + 0x50, 0x52, 0x53, 0x55, 0x57, 0x59, 0x5a, 0x5c, + 0x5e, 0x5f, 0x61, 0x62, 0x63, 0x65, 0x66, 0x67, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, + 0x71, 0x72, 0x72, 0x73, 0x74, 0x74, 0x75, 0x76, + 0x76, 0x77, 0x77, 0x78, 0x78, 0x79, 0x79, 0x7a, + 0x7a, 0x7a, 0x7b, 0x7b, 0x7b, 0x7c, 0x7c, 0x7c, + 0x7c, 0x7c, 0x7d, 0x7d, 0x7d, 0x7d, 0x7d, 0x7e, + 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, + 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, + 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x04, + 0x04, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, + 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, 0x09, + 0x0a, 0x0a, 0x0b, 0x0c, 0x0c, 0x0d, 0x0e, 0x0e, + 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, + 0x17, 0x19, 0x1a, 0x1b, 0x1d, 0x1e, 0x1f, 0x21, + 0x22, 0x24, 0x26, 0x27, 0x29, 0x2b, 0x2d, 0x2e, + 0x30, 0x32, 0x34, 0x36, 0x38, 0x3a, 0x3c, 0x3e, +}; + +const q15_t sigmoidTable_q15[256] = { + 0x4000, 0x4200, 0x43ff, 0x45fc, 0x47f5, 0x49eb, 0x4bdc, 0x4dc8, + 0x4fad, 0x518a, 0x5360, 0x552c, 0x56ef, 0x58a8, 0x5a57, 0x5bfb, + 0x5d93, 0x5f20, 0x60a1, 0x6216, 0x637f, 0x64db, 0x662b, 0x676f, + 0x68a6, 0x69d2, 0x6af1, 0x6c05, 0x6d0d, 0x6e09, 0x6efb, 0x6fe2, + 0x70be, 0x7190, 0x7258, 0x7316, 0x73cc, 0x7478, 0x751b, 0x75b7, + 0x764a, 0x76d6, 0x775b, 0x77d8, 0x784f, 0x78c0, 0x792a, 0x798f, + 0x79ee, 0x7a48, 0x7a9d, 0x7aed, 0x7b39, 0x7b80, 0x7bc4, 0x7c03, + 0x7c3f, 0x7c78, 0x7cad, 0x7ce0, 0x7d0f, 0x7d3c, 0x7d66, 0x7d8d, + 0x7db3, 0x7dd6, 0x7df7, 0x7e16, 0x7e33, 0x7e4f, 0x7e69, 0x7e81, + 0x7e98, 0x7eae, 0x7ec2, 0x7ed5, 0x7ee7, 0x7ef8, 0x7f08, 0x7f17, + 0x7f25, 0x7f32, 0x7f3e, 0x7f4a, 0x7f55, 0x7f5f, 0x7f69, 0x7f72, + 0x7f7b, 0x7f83, 0x7f8a, 0x7f91, 0x7f98, 0x7f9e, 0x7fa4, 0x7faa, + 0x7faf, 0x7fb4, 0x7fb8, 0x7fbd, 0x7fc1, 0x7fc5, 0x7fc8, 0x7fcc, + 0x7fcf, 0x7fd2, 0x7fd5, 0x7fd7, 0x7fda, 0x7fdc, 0x7fde, 0x7fe0, + 0x7fe2, 0x7fe4, 0x7fe6, 0x7fe7, 0x7fe9, 0x7fea, 0x7feb, 0x7fed, + 0x7fee, 0x7fef, 0x7ff0, 0x7ff1, 0x7ff2, 0x7ff3, 0x7ff4, 0x7ff4, + 0x000b, 0x000c, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, + 0x0012, 0x0013, 0x0015, 0x0016, 0x0017, 0x0019, 0x001a, 0x001c, + 0x001e, 0x0020, 0x0022, 0x0024, 0x0026, 0x0029, 0x002b, 0x002e, + 0x0031, 0x0034, 0x0038, 0x003b, 0x003f, 0x0043, 0x0048, 0x004c, + 0x0051, 0x0056, 0x005c, 0x0062, 0x0068, 0x006f, 0x0076, 0x007d, + 0x0085, 0x008e, 0x0097, 0x00a1, 0x00ab, 0x00b6, 0x00c2, 0x00ce, + 0x00db, 0x00e9, 0x00f8, 0x0108, 0x0119, 0x012b, 0x013e, 0x0152, + 0x0168, 0x017f, 0x0197, 0x01b1, 0x01cd, 0x01ea, 0x0209, 0x022a, + 0x024d, 0x0273, 0x029a, 0x02c4, 0x02f1, 0x0320, 0x0353, 0x0388, + 0x03c1, 0x03fd, 0x043c, 0x0480, 0x04c7, 0x0513, 0x0563, 0x05b8, + 0x0612, 0x0671, 0x06d6, 0x0740, 0x07b1, 0x0828, 0x08a5, 0x092a, + 0x09b6, 0x0a49, 0x0ae5, 0x0b88, 0x0c34, 0x0cea, 0x0da8, 0x0e70, + 0x0f42, 0x101e, 0x1105, 0x11f7, 0x12f3, 0x13fb, 0x150f, 0x162e, + 0x175a, 0x1891, 0x19d5, 0x1b25, 0x1c81, 0x1dea, 0x1f5f, 0x20e0, + 0x226d, 0x2405, 0x25a9, 0x2758, 0x2911, 0x2ad4, 0x2ca0, 0x2e76, + 0x3053, 0x3238, 0x3424, 0x3615, 0x380b, 0x3a04, 0x3c01, 0x3e00, +}; + +const q15_t sigmoidLTable_q15[128] = { + 0x4000, 0x4100, 0x4200, 0x42ff, 0x43ff, 0x44fd, 0x45fc, 0x46f9, + 0x47f5, 0x48f1, 0x49eb, 0x4ae5, 0x4bdc, 0x4cd3, 0x4dc8, 0x4ebb, + 0x4fad, 0x509c, 0x518a, 0x5276, 0x5360, 0x5447, 0x552c, 0x560f, + 0x56ef, 0x57cd, 0x58a8, 0x5981, 0x5a57, 0x5b2a, 0x5bfb, 0x5cc9, + 0x5d93, 0x5e5b, 0x5f20, 0x5fe2, 0x60a1, 0x615d, 0x6216, 0x62cc, + 0x637f, 0x642e, 0x64db, 0x6584, 0x662b, 0x66ce, 0x676f, 0x680c, + 0x68a6, 0x693d, 0x69d2, 0x6a63, 0x6af1, 0x6b7c, 0x6c05, 0x6c8a, + 0x6d0d, 0x6d8d, 0x6e09, 0x6e84, 0x6efb, 0x6f70, 0x6fe2, 0x7051, + 0x0f42, 0x0faf, 0x101e, 0x1090, 0x1105, 0x117c, 0x11f7, 0x1273, + 0x12f3, 0x1376, 0x13fb, 0x1484, 0x150f, 0x159d, 0x162e, 0x16c3, + 0x175a, 0x17f4, 0x1891, 0x1932, 0x19d5, 0x1a7c, 0x1b25, 0x1bd2, + 0x1c81, 0x1d34, 0x1dea, 0x1ea3, 0x1f5f, 0x201e, 0x20e0, 0x21a5, + 0x226d, 0x2337, 0x2405, 0x24d6, 0x25a9, 0x267f, 0x2758, 0x2833, + 0x2911, 0x29f1, 0x2ad4, 0x2bb9, 0x2ca0, 0x2d8a, 0x2e76, 0x2f64, + 0x3053, 0x3145, 0x3238, 0x332d, 0x3424, 0x351b, 0x3615, 0x370f, + 0x380b, 0x3907, 0x3a04, 0x3b03, 0x3c01, 0x3d01, 0x3e00, 0x3f00, +}; + +const q15_t sigmoidHTable_q15[192] = { + 0x70be, 0x7190, 0x7258, 0x7316, 0x73cc, 0x7478, 0x751b, 0x75b7, + 0x764a, 0x76d6, 0x775b, 0x77d8, 0x784f, 0x78c0, 0x792a, 0x798f, + 0x79ee, 0x7a48, 0x7a9d, 0x7aed, 0x7b39, 0x7b80, 0x7bc4, 0x7c03, + 0x7c3f, 0x7c78, 0x7cad, 0x7ce0, 0x7d0f, 0x7d3c, 0x7d66, 0x7d8d, + 0x7db3, 0x7dd6, 0x7df7, 0x7e16, 0x7e33, 0x7e4f, 0x7e69, 0x7e81, + 0x7e98, 0x7eae, 0x7ec2, 0x7ed5, 0x7ee7, 0x7ef8, 0x7f08, 0x7f17, + 0x7f25, 0x7f32, 0x7f3e, 0x7f4a, 0x7f55, 0x7f5f, 0x7f69, 0x7f72, + 0x7f7b, 0x7f83, 0x7f8a, 0x7f91, 0x7f98, 0x7f9e, 0x7fa4, 0x7faa, + 0x7faf, 0x7fb4, 0x7fb8, 0x7fbd, 0x7fc1, 0x7fc5, 0x7fc8, 0x7fcc, + 0x7fcf, 0x7fd2, 0x7fd5, 0x7fd7, 0x7fda, 0x7fdc, 0x7fde, 0x7fe0, + 0x7fe2, 0x7fe4, 0x7fe6, 0x7fe7, 0x7fe9, 0x7fea, 0x7feb, 0x7fed, + 0x7fee, 0x7fef, 0x7ff0, 0x7ff1, 0x7ff2, 0x7ff3, 0x7ff4, 0x7ff4, + 0x000b, 0x000c, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, + 0x0012, 0x0013, 0x0015, 0x0016, 0x0017, 0x0019, 0x001a, 0x001c, + 0x001e, 0x0020, 0x0022, 0x0024, 0x0026, 0x0029, 0x002b, 0x002e, + 0x0031, 0x0034, 0x0038, 0x003b, 0x003f, 0x0043, 0x0048, 0x004c, + 0x0051, 0x0056, 0x005c, 0x0062, 0x0068, 0x006f, 0x0076, 0x007d, + 0x0085, 0x008e, 0x0097, 0x00a1, 0x00ab, 0x00b6, 0x00c2, 0x00ce, + 0x00db, 0x00e9, 0x00f8, 0x0108, 0x0119, 0x012b, 0x013e, 0x0152, + 0x0168, 0x017f, 0x0197, 0x01b1, 0x01cd, 0x01ea, 0x0209, 0x022a, + 0x024d, 0x0273, 0x029a, 0x02c4, 0x02f1, 0x0320, 0x0353, 0x0388, + 0x03c1, 0x03fd, 0x043c, 0x0480, 0x04c7, 0x0513, 0x0563, 0x05b8, + 0x0612, 0x0671, 0x06d6, 0x0740, 0x07b1, 0x0828, 0x08a5, 0x092a, + 0x09b6, 0x0a49, 0x0ae5, 0x0b88, 0x0c34, 0x0cea, 0x0da8, 0x0e70, +}; + +const q7_t tanhTable_q7[256] = { + 0x00, 0x08, 0x10, 0x18, 0x1f, 0x27, 0x2e, 0x35, + 0x3b, 0x41, 0x47, 0x4c, 0x51, 0x56, 0x5a, 0x5e, + 0x61, 0x65, 0x68, 0x6a, 0x6d, 0x6f, 0x71, 0x72, + 0x74, 0x75, 0x76, 0x78, 0x78, 0x79, 0x7a, 0x7b, + 0x7b, 0x7c, 0x7c, 0x7d, 0x7d, 0x7e, 0x7e, 0x7e, + 0x7e, 0x7e, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x81, + 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x82, + 0x82, 0x82, 0x82, 0x82, 0x83, 0x83, 0x84, 0x84, + 0x85, 0x85, 0x86, 0x87, 0x88, 0x88, 0x8a, 0x8b, + 0x8c, 0x8e, 0x8f, 0x91, 0x93, 0x96, 0x98, 0x9b, + 0x9f, 0xa2, 0xa6, 0xaa, 0xaf, 0xb4, 0xb9, 0xbf, + 0xc5, 0xcb, 0xd2, 0xd9, 0xe1, 0xe8, 0xf0, 0xf8, +}; + +const q15_t tanhTable_q15[256] = { + 0x0000, 0x07fd, 0x0feb, 0x17b9, 0x1f59, 0x26bf, 0x2ddf, 0x34ae, + 0x3b27, 0x4142, 0x46fd, 0x4c56, 0x514d, 0x55e2, 0x5a1a, 0x5df6, + 0x617c, 0x64b0, 0x6797, 0x6a37, 0x6c95, 0x6eb5, 0x709e, 0x7254, + 0x73dc, 0x753a, 0x7672, 0x7788, 0x787f, 0x795b, 0x7a1e, 0x7acb, + 0x7b65, 0x7bee, 0x7c66, 0x7cd1, 0x7d30, 0x7d84, 0x7dce, 0x7e0f, + 0x7e49, 0x7e7d, 0x7eaa, 0x7ed2, 0x7ef5, 0x7f14, 0x7f30, 0x7f48, + 0x7f5e, 0x7f71, 0x7f82, 0x7f91, 0x7f9e, 0x7fa9, 0x7fb3, 0x7fbc, + 0x7fc4, 0x7fcb, 0x7fd1, 0x7fd7, 0x7fdc, 0x7fe0, 0x7fe4, 0x7fe7, + 0x7fea, 0x7fed, 0x7fef, 0x7ff1, 0x7ff3, 0x7ff4, 0x7ff6, 0x7ff7, + 0x7ff8, 0x7ff9, 0x7ffa, 0x7ffa, 0x7ffb, 0x7ffc, 0x7ffc, 0x7ffd, + 0x7ffd, 0x7ffd, 0x7ffe, 0x7ffe, 0x7ffe, 0x7ffe, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, + 0x8001, 0x8001, 0x8001, 0x8002, 0x8002, 0x8002, 0x8002, 0x8003, + 0x8003, 0x8003, 0x8004, 0x8004, 0x8005, 0x8006, 0x8006, 0x8007, + 0x8008, 0x8009, 0x800a, 0x800c, 0x800d, 0x800f, 0x8011, 0x8013, + 0x8016, 0x8019, 0x801c, 0x8020, 0x8024, 0x8029, 0x802f, 0x8035, + 0x803c, 0x8044, 0x804d, 0x8057, 0x8062, 0x806f, 0x807e, 0x808f, + 0x80a2, 0x80b8, 0x80d0, 0x80ec, 0x810b, 0x812e, 0x8156, 0x8183, + 0x81b7, 0x81f1, 0x8232, 0x827c, 0x82d0, 0x832f, 0x839a, 0x8412, + 0x849b, 0x8535, 0x85e2, 0x86a5, 0x8781, 0x8878, 0x898e, 0x8ac6, + 0x8c24, 0x8dac, 0x8f62, 0x914b, 0x936b, 0x95c9, 0x9869, 0x9b50, + 0x9e84, 0xa20a, 0xa5e6, 0xaa1e, 0xaeb3, 0xb3aa, 0xb903, 0xbebe, + 0xc4d9, 0xcb52, 0xd221, 0xd941, 0xe0a7, 0xe847, 0xf015, 0xf803, +}; + +const q15_t tanhLTable_q15[128] = { + 0x0000, 0x0400, 0x07fd, 0x0bf7, 0x0feb, 0x13d7, 0x17b9, 0x1b90, + 0x1f59, 0x2314, 0x26bf, 0x2a58, 0x2ddf, 0x3151, 0x34ae, 0x37f6, + 0x3b27, 0x3e40, 0x4142, 0x442c, 0x46fd, 0x49b6, 0x4c56, 0x4edd, + 0x514d, 0x53a3, 0x55e2, 0x580a, 0x5a1a, 0x5c13, 0x5df6, 0x5fc4, + 0x617c, 0x6320, 0x64b0, 0x662d, 0x6797, 0x68f0, 0x6a37, 0x6b6e, + 0x6c95, 0x6dac, 0x6eb5, 0x6fb0, 0x709e, 0x717f, 0x7254, 0x731e, + 0x73dc, 0x7490, 0x753a, 0x75da, 0x7672, 0x7701, 0x7788, 0x7807, + 0x787f, 0x78f0, 0x795b, 0x79bf, 0x7a1e, 0x7a77, 0x7acb, 0x7b1b, + 0x849b, 0x84e5, 0x8535, 0x8589, 0x85e2, 0x8641, 0x86a5, 0x8710, + 0x8781, 0x87f9, 0x8878, 0x88ff, 0x898e, 0x8a26, 0x8ac6, 0x8b70, + 0x8c24, 0x8ce2, 0x8dac, 0x8e81, 0x8f62, 0x9050, 0x914b, 0x9254, + 0x936b, 0x9492, 0x95c9, 0x9710, 0x9869, 0x99d3, 0x9b50, 0x9ce0, + 0x9e84, 0xa03c, 0xa20a, 0xa3ed, 0xa5e6, 0xa7f6, 0xaa1e, 0xac5d, + 0xaeb3, 0xb123, 0xb3aa, 0xb64a, 0xb903, 0xbbd4, 0xbebe, 0xc1c0, + 0xc4d9, 0xc80a, 0xcb52, 0xceaf, 0xd221, 0xd5a8, 0xd941, 0xdcec, + 0xe0a7, 0xe470, 0xe847, 0xec29, 0xf015, 0xf409, 0xf803, 0xfc00, +}; + +const q15_t tanhHTable_q15[192] = { + 0x7b65, 0x7bee, 0x7c66, 0x7cd1, 0x7d30, 0x7d84, 0x7dce, 0x7e0f, + 0x7e49, 0x7e7d, 0x7eaa, 0x7ed2, 0x7ef5, 0x7f14, 0x7f30, 0x7f48, + 0x7f5e, 0x7f71, 0x7f82, 0x7f91, 0x7f9e, 0x7fa9, 0x7fb3, 0x7fbc, + 0x7fc4, 0x7fcb, 0x7fd1, 0x7fd7, 0x7fdc, 0x7fe0, 0x7fe4, 0x7fe7, + 0x7fea, 0x7fed, 0x7fef, 0x7ff1, 0x7ff3, 0x7ff4, 0x7ff6, 0x7ff7, + 0x7ff8, 0x7ff9, 0x7ffa, 0x7ffa, 0x7ffb, 0x7ffc, 0x7ffc, 0x7ffd, + 0x7ffd, 0x7ffd, 0x7ffe, 0x7ffe, 0x7ffe, 0x7ffe, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, + 0x8001, 0x8001, 0x8001, 0x8002, 0x8002, 0x8002, 0x8002, 0x8003, + 0x8003, 0x8003, 0x8004, 0x8004, 0x8005, 0x8006, 0x8006, 0x8007, + 0x8008, 0x8009, 0x800a, 0x800c, 0x800d, 0x800f, 0x8011, 0x8013, + 0x8016, 0x8019, 0x801c, 0x8020, 0x8024, 0x8029, 0x802f, 0x8035, + 0x803c, 0x8044, 0x804d, 0x8057, 0x8062, 0x806f, 0x807e, 0x808f, + 0x80a2, 0x80b8, 0x80d0, 0x80ec, 0x810b, 0x812e, 0x8156, 0x8183, + 0x81b7, 0x81f1, 0x8232, 0x827c, 0x82d0, 0x832f, 0x839a, 0x8412, +}; diff --git a/NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c b/NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c new file mode 100644 index 0000000..264e760 --- /dev/null +++ b/NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_q7_to_q15_no_shift.c + * Description: Converts the elements of the Q7 vector to Q15 vector without left-shift + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_nnsupportfunctions.h" + +/** + * @ingroup groupSupport + */ + +/** + * @addtogroup nndata_convert + * @{ + */ + +/** + * @brief Converts the elements of the Q7 vector to Q15 vector without left-shift + * @param[in] *pSrc points to the Q7 input vector + * @param[out] *pDst points to the Q15 output vector + * @param[in] blockSize length of the input vector + * @return none. + * + * \par Description: + * + * The equation used for the conversion process is: + * + * 
    
+ * 	pDst[n] = (q15_t) pSrc[n];   0 <= n < blockSize.    
+ *
+ * + */ + +void arm_q7_to_q15_no_shift(const q7_t * pSrc, q15_t * pDst, uint32_t blockSize) +{ + const q7_t *pIn = pSrc; /* Src pointer */ + uint32_t blkCnt; /* loop counter */ + +#ifndef ARM_MATH_CM0_FAMILY + q31_t in; + q31_t in1, in2; + q31_t out1, out2; + + /* Run the below code for Cortex-M4 and Cortex-M3 */ + + /*loop Unrolling */ + blkCnt = blockSize >> 2u; + + /* First part of the processing with loop unrolling. Compute 4 outputs at a time. + ** a second loop below computes the remaining 1 to 3 samples. */ + while (blkCnt > 0u) + { + /* C = (q15_t) A << 8 */ + /* convert from q7 to q15 and then store the results in the destination buffer */ + in = *__SIMD32(pIn)++; + + /* rotatate in by 8 and extend two q7_t values to q15_t values */ + in1 = __SXTB16(__ROR(in, 8)); + + /* extend remainig two q7_t values to q15_t values */ + in2 = __SXTB16(in); + +#ifndef ARM_MATH_BIG_ENDIAN + + out2 = __PKHTB(in1, in2, 16); + out1 = __PKHBT(in2, in1, 16); + +#else + + out1 = __PKHTB(in1, in2, 16); + out2 = __PKHBT(in2, in1, 16); + +#endif + + *__SIMD32(pDst)++ = out1; + *__SIMD32(pDst)++ = out2; + + /* Decrement the loop counter */ + blkCnt--; + } + + /* If the blockSize is not a multiple of 4, compute any remaining output samples here. + ** No loop unrolling is used. */ + blkCnt = blockSize % 0x4u; + +#else + + /* Run the below code for Cortex-M0 */ + + /* Loop over blockSize number of values */ + blkCnt = blockSize; + +#endif /* #ifndef ARM_MATH_CM0_FAMILY */ + + while (blkCnt > 0u) + { + /* C = (q15_t) A << 8 */ + /* convert from q7 to q15 and then store the results in the destination buffer */ + *pDst++ = (q15_t) * pIn++; + + /* Decrement the loop counter */ + blkCnt--; + } + +} + +/** + * @} end of nndata_convert group + */ diff --git a/NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c b/NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c new file mode 100644 index 0000000..7d29aa4 --- /dev/null +++ b/NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_q7_to_q15_reordered_no_shift.c + * Description: Converts the elements of the Q7 vector to reordered Q15 vector without left-shift + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_nnsupportfunctions.h" + +/** + * @ingroup groupSupport + */ + +/** + * @addtogroup nndata_convert + * @{ + */ + +/** + * @brief Converts the elements of the Q7 vector to reordered Q15 vector without left-shift + * @param[in] *pSrc points to the Q7 input vector + * @param[out] *pDst points to the Q15 output vector + * @param[in] blockSize length of the input vector + * @return none. + * + * @details + * + * This function does the q7 to q15 expansion with re-ordering + * + * 
+ *                          |   A1   |   A2   |   A3   |   A4   |
+ *
+ *                           0      7 8     15 16    23 24    31
+ *
+ * + * is converted into: + * + * 
+ *  |       A1       |       A3       |   and  |       A2       |       A4       |
+ *
+ *   0             15 16            31          0             15 16            31
+ *
+ * + * + * This looks strange but is natural considering how sign-extension is done at + * assembly level. + * + * The expansion of other other oprand will follow the same rule so that the end + * results are the same. + * + * The tail (i.e., last (N % 4) elements) will still be in original order. + * + */ + +void arm_q7_to_q15_reordered_no_shift(const q7_t * pSrc, q15_t * pDst, uint32_t blockSize) +{ + const q7_t *pIn = pSrc; /* Src pointer */ + uint32_t blkCnt; /* loop counter */ + +#ifndef ARM_MATH_CM0_FAMILY + q31_t in; + q31_t in1, in2; + + /* Run the below code for Cortex-M4 and Cortex-M3 */ + + /*loop Unrolling */ + blkCnt = blockSize >> 2u; + + /* First part of the processing with loop unrolling. Compute 4 outputs at a time. + ** a second loop below computes the remaining 1 to 3 samples. */ + while (blkCnt > 0u) + { + /* C = (q15_t) A << 8 */ + /* convert from q7 to q15 and then store the results in the destination buffer */ + in = *__SIMD32(pIn)++; + + /* rotatate in by 8 and extend two q7_t values to q15_t values */ + in1 = __SXTB16(__ROR(in, 8)); + + /* extend remainig two q7_t values to q15_t values */ + in2 = __SXTB16(in); + +#ifndef ARM_MATH_BIG_ENDIAN + *__SIMD32(pDst)++ = in2; + *__SIMD32(pDst)++ = in1; +#else + *__SIMD32(pDst)++ = in1; + *__SIMD32(pDst)++ = in2; +#endif + + /* Decrement the loop counter */ + blkCnt--; + } + + /* If the blockSize is not a multiple of 4, compute any remaining output samples here. + ** No loop unrolling is used. */ + blkCnt = blockSize % 0x4u; + +#else + + /* Run the below code for Cortex-M0 */ + + /* Loop over blockSize number of values */ + blkCnt = blockSize; + +#endif /* #ifndef ARM_MATH_CM0_FAMILY */ + + while (blkCnt > 0u) + { + /* C = (q15_t) A << 8 */ + /* convert from q7 to q15 and then store the results in the destination buffer */ + *pDst++ = (q15_t) * pIn++; + + /* Decrement the loop counter */ + blkCnt--; + } + +} + +/** + * @} end of q7_to_x group + */ diff --git a/NN/Source/PoolingFunctions/arm_pool_q7_HWC.c b/NN/Source/PoolingFunctions/arm_pool_q7_HWC.c new file mode 100644 index 0000000..b451f5e --- /dev/null +++ b/NN/Source/PoolingFunctions/arm_pool_q7_HWC.c @@ -0,0 +1,448 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_pool_q7_HWC.c + * Description: Pooling function implementations + * + * $Date: 17. January 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +#if defined (ARM_MATH_DSP) + +/** + * @brief A few utility functions used by pooling functions + * + * + */ + +static void buffer_scale_back_q15_to_q7(q15_t * buffer, q7_t * target, uint16_t length, uint16_t scale) +{ + int i; + + for (i = 0; i < length; i++) + { + target[i] = (q7_t) (buffer[i] / scale); + } +} + +static void compare_and_replace_if_larger_q7(q7_t * base, // base data + q7_t * target, // compare target + const uint16_t length // data length + ) +{ + q7_t *pIn = base; + q7_t *pCom = target; + union arm_nnword in; + union arm_nnword com; + uint16_t cnt = length >> 2; + + while (cnt > 0u) + { + in.word = *__SIMD32(pIn); + com.word = *__SIMD32(pCom)++; + + // if version + if (com.bytes[0] > in.bytes[0]) + in.bytes[0] = com.bytes[0]; + if (com.bytes[1] > in.bytes[1]) + in.bytes[1] = com.bytes[1]; + if (com.bytes[2] > in.bytes[2]) + in.bytes[2] = com.bytes[2]; + if (com.bytes[3] > in.bytes[3]) + in.bytes[3] = com.bytes[3]; + + *__SIMD32(pIn)++ = in.word; + + cnt--; + } +} + +static void accumulate_q7_to_q15(q15_t * base, q7_t * target, const uint16_t length) +{ + q15_t *pCnt = base; + q7_t *pV = target; + q31_t v1, v2, vo1, vo2; + uint16_t cnt = length >> 2; + q31_t in; + + while (cnt > 0u) + { + q31_t value = *__SIMD32(pV)++; + v1 = __SXTB16(__ROR(value, 8)); + v2 = __SXTB16(value); +#ifndef ARM_MATH_BIG_ENDIAN + + vo2 = __PKHTB(v1, v2, 16); + vo1 = __PKHBT(v2, v1, 16); + +#else + + vo1 = __PKHTB(v1, v2, 16); + vo2 = __PKHBT(v2, v1, 16); + +#endif + + in = *__SIMD32(pCnt); + *__SIMD32(pCnt)++ = __QADD16(vo1, in); + + in = *__SIMD32(pCnt); + *__SIMD32(pCnt)++ = __QADD16(vo2, in); + + cnt--; + } + cnt = length & 0x3; + while (cnt > 0u) + { + *pCnt++ += *pV++; + cnt--; + } +} + +#endif // ARM_MATH_DSP + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup Pooling + * @{ + */ + + /** + * @brief Q7 max pooling function + * @param[in, out] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] Im_out pointer to output tensor + * @return none. + * + * @details + * + * Buffer size: + * + * bufferA size: 0 + * + * The pooling function is implemented as split x-pooling then + * y-pooling. + * + * This pooling function is input-destructive. Input data is undefined + * after calling this function. + * + */ + +void +arm_maxpool_q7_HWC(q7_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, const uint16_t dim_im_out, q7_t * bufferA, q7_t * Im_out) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + int16_t i_x, i_y; + + /* first does the pooling along x axis */ + for (i_y = 0; i_y < dim_im_in; i_y++) + { + + for (i_x = 0; i_x < dim_im_out; i_x++) + { + /* for each output pixel */ + q7_t *target = Im_in + (i_y * dim_im_in + i_x) * ch_im_in; + q7_t *win_start; + q7_t *win_stop; + if (i_x * stride - padding < 0) + { + win_start = target; + } else + { + win_start = Im_in + (i_y * dim_im_in + i_x * stride - padding) * ch_im_in; + } + + if (i_x * stride - padding + dim_kernel >= dim_im_in) + { + win_stop = Im_in + (i_y * dim_im_in + dim_im_in) * ch_im_in; + } else + { + win_stop = Im_in + (i_y * dim_im_in + i_x * stride - padding + dim_kernel) * ch_im_in; + } + + /* first step is to copy over initial data */ + /* arm_copy_q7(win_start, target, ch_im_in); */ + memmove(target, win_start, ch_im_in); + + /* start the max operation from the second part */ + win_start += ch_im_in; + for (; win_start < win_stop; win_start += ch_im_in) + { + compare_and_replace_if_larger_q7(target, win_start, ch_im_in); + } + } + } + + /* then does the pooling along y axis */ + for (i_y = 0; i_y < dim_im_out; i_y++) + { + + /* for each output row */ + q7_t *target = Im_out + i_y * dim_im_out * ch_im_in; + q7_t *row_start; + q7_t *row_end; + /* setting the starting row */ + if (i_y * stride - padding < 0) + { + row_start = Im_in; + } else + { + row_start = Im_in + (i_y * stride - padding) * dim_im_in * ch_im_in; + } + /* setting the stopping row */ + if (i_y * stride - padding + dim_kernel >= dim_im_in) + { + row_end = Im_in + dim_im_in * dim_im_in * ch_im_in; + } else + { + row_end = Im_in + (i_y * stride - padding + dim_kernel) * dim_im_in * ch_im_in; + } + + /* copy over the first row */ + /* arm_copy_q7(row_start, target, dim_im_out * ch_im_in); */ + memmove(target, row_start, dim_im_out * ch_im_in); + + /* move over to next row */ + row_start += ch_im_in * dim_im_in; + + for (; row_start < row_end; row_start += dim_im_in * ch_im_in) + { + compare_and_replace_if_larger_q7(target, row_start, dim_im_out * ch_im_in); + } + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + int16_t i_ch_in, i_x, i_y; + int16_t k_x, k_y; + + for (i_ch_in = 0; i_ch_in < ch_im_in; i_ch_in++) + { + for (i_y = 0; i_y < dim_im_out; i_y++) + { + for (i_x = 0; i_x < dim_im_out; i_x++) + { + int max = -129; + for (k_y = i_y * stride - padding; k_y < i_y * stride - padding + dim_kernel; k_y++) + { + for (k_x = i_x * stride - padding; k_x < i_x * stride - padding + dim_kernel; k_x++) + { + if (k_y >= 0 && k_x >= 0 && k_y < dim_im_in && k_x < dim_im_in) + { + if (Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)] > max) + { + max = Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)]; + } + } + } + } + Im_out[i_ch_in + ch_im_in * (i_x + i_y * dim_im_out)] = max; + } + } + } + +#endif /* ARM_MATH_DSP */ + +} + + /** + * @brief Q7 average pooling function + * @param[in,out] Im_in pointer to input tensor + * @param[in] dim_im_in input tensor dimention + * @param[in] ch_im_in number of input tensor channels + * @param[in] dim_kernel filter kernel size + * @param[in] padding padding sizes + * @param[in] stride convolution stride + * @param[in] dim_im_out output tensor dimension + * @param[in,out] bufferA pointer to buffer space for input + * @param[in,out] Im_out pointer to output tensor + * @return none. + * + * @details + * + * Buffer size: + * + * bufferA size: 2*dim_im_out*ch_im_in + * + * The pooling function is implemented as split x-pooling then + * y-pooling. + * + * This pooling function is input-destructive. Input data is undefined + * after calling this function. + * + */ + +void +arm_avepool_q7_HWC(q7_t * Im_in, + const uint16_t dim_im_in, + const uint16_t ch_im_in, + const uint16_t dim_kernel, + const uint16_t padding, + const uint16_t stride, const uint16_t dim_im_out, q7_t * bufferA, q7_t * Im_out) +{ + +#if defined (ARM_MATH_DSP) + /* Run the following code for Cortex-M4 and Cortex-M7 */ + + q15_t *buffer = (q15_t *) bufferA; + int16_t i_x, i_y; + int16_t count = 0; + + /* first does the pooling along x axis */ + for (i_y = 0; i_y < dim_im_in; i_y++) + { + + for (i_x = 0; i_x < dim_im_out; i_x++) + { + /* for each output pixel */ + q7_t *target = Im_in + (i_y * dim_im_in + i_x) * ch_im_in; + q7_t *win_start; + q7_t *win_stop; + if (i_x * stride - padding < 0) + { + win_start = target; + } else + { + win_start = Im_in + (i_y * dim_im_in + i_x * stride - padding) * ch_im_in; + } + + if (i_x * stride - padding + dim_kernel >= dim_im_in) + { + win_stop = Im_in + (i_y * dim_im_in + dim_im_in) * ch_im_in; + } else + { + win_stop = Im_in + (i_y * dim_im_in + i_x * stride - padding + dim_kernel) * ch_im_in; + } + + /* first step is to copy over initial data */ + arm_q7_to_q15_no_shift(win_start, buffer, ch_im_in); + count = 1; + + /* start the max operation from the second part */ + win_start += ch_im_in; + for (; win_start < win_stop; win_start += ch_im_in) + { + accumulate_q7_to_q15(buffer, win_start, ch_im_in); + count++; + } + buffer_scale_back_q15_to_q7(buffer, target, ch_im_in, count); + } + } + + /* then does the pooling along y axis */ + for (i_y = 0; i_y < dim_im_out; i_y++) + { + /* for each output row */ + q7_t *target = Im_out + i_y * dim_im_out * ch_im_in; + q7_t *row_start; + q7_t *row_end; + /* setting the starting row */ + if (i_y * stride - padding < 0) + { + row_start = Im_in; + } else + { + row_start = Im_in + (i_y * stride - padding) * dim_im_in * ch_im_in; + } + /* setting the stopping row */ + if (i_y * stride - padding + dim_kernel >= dim_im_in) + { + row_end = Im_in + dim_im_in * dim_im_in * ch_im_in; + } else + { + row_end = Im_in + (i_y * stride - padding + dim_kernel) * dim_im_in * ch_im_in; + } + + /* copy over the first row */ + arm_q7_to_q15_no_shift(row_start, buffer, dim_im_out * ch_im_in); + count = 1; + + /* move over to next row */ + row_start += ch_im_in * dim_im_in; + + for (; row_start < row_end; row_start += dim_im_in * ch_im_in) + { + accumulate_q7_to_q15(buffer, row_start, dim_im_out * ch_im_in); + count++; + } + buffer_scale_back_q15_to_q7(buffer, target, dim_im_out * ch_im_in, count); + } + +#else + /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ + + int16_t i_ch_in, i_x, i_y; + int16_t k_x, k_y; + + for (i_ch_in = 0; i_ch_in < ch_im_in; i_ch_in++) + { + for (i_y = 0; i_y < dim_im_out; i_y++) + { + for (i_x = 0; i_x < dim_im_out; i_x++) + { + int sum = 0; + int count = 0; + for (k_y = i_y * stride - padding; k_y < i_y * stride - padding + dim_kernel; k_y++) + { + for (k_x = i_x * stride - padding; k_x < i_x * stride - padding + dim_kernel; k_x++) + { + if (k_y >= 0 && k_x >= 0 && k_y < dim_im_in && k_x < dim_im_in) + { + sum += Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)]; + count++; + } + } + } + Im_out[i_ch_in + ch_im_in * (i_x + i_y * dim_im_out)] = sum / count; + } + } + } + +#endif /* ARM_MATH_DSP */ + +} + +/** + * @} end of Pooling group + */ diff --git a/NN/Source/SoftmaxFunctions/arm_softmax_q15.c b/NN/Source/SoftmaxFunctions/arm_softmax_q15.c new file mode 100644 index 0000000..abc2737 --- /dev/null +++ b/NN/Source/SoftmaxFunctions/arm_softmax_q15.c @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_softmax_q15.c + * Description: Q15 softmax function + * + * $Date: 20. February 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup Softmax + * @{ + */ + + /** + * @brief Q15 softmax function + * @param[in] vec_in pointer to input vector + * @param[in] dim_vec input vector dimention + * @param[out] p_out pointer to output vector + * @return none. + * + * @details + * + * Here, instead of typical e based softmax, we use + * 2-based softmax, i.e.,: + * + * y_i = 2^(x_i) / sum(2^x_j) + * + * The relative output will be different here. + * But mathematically, the gradient will be the same + * with a log(2) scaling factor. + * + */ + +void arm_softmax_q15(const q15_t * vec_in, const uint16_t dim_vec, q15_t * p_out) +{ + q31_t sum; + int16_t i; + uint8_t shift; + q31_t base; + base = -1 * 0x100000; + for (i = 0; i < dim_vec; i++) + { + if (vec_in[i] > base) + { + base = vec_in[i]; + } + } + + /* we ignore really small values + * anyway, they will be 0 after shrinking + * to q15_t + */ + base = base - 16; + + sum = 0; + + for (i = 0; i < dim_vec; i++) + { + if (vec_in[i] > base) + { + shift = (uint8_t)__USAT(vec_in[i] - base, 5); + sum += 0x1 << shift; + } + } + + /* This is effectively (0x1 << 32) / sum */ + int64_t div_base = 0x100000000LL; + int output_base = (int32_t)(div_base / sum); + + /* Final confidence will be output_base >> ( 17 - (vec_in[i] - base) ) + * so 32768 (0x1<<15) -> 100% confidence when sum = 0x1 << 16, output_base = 0x1 << 16 + * and vec_in[i]-base = 16 + */ + for (i = 0; i < dim_vec; i++) + { + if (vec_in[i] > base) + { + /* Here minimum value of 17+base-vec[i] will be 1 */ + shift = (uint8_t)__USAT(17+base-vec_in[i], 5); + p_out[i] = (q15_t) __SSAT((output_base >> shift), 16); + } else + { + p_out[i] = 0; + } + } + +} + +/** + * @} end of Softmax group + */ diff --git a/NN/Source/SoftmaxFunctions/arm_softmax_q7.c b/NN/Source/SoftmaxFunctions/arm_softmax_q7.c new file mode 100644 index 0000000..a4e2548 --- /dev/null +++ b/NN/Source/SoftmaxFunctions/arm_softmax_q7.c @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the License); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* ---------------------------------------------------------------------- + * Project: CMSIS NN Library + * Title: arm_softmax_q7.c + * Description: Q7 softmax function + * + * $Date: 20. February 2018 + * $Revision: V.1.0.0 + * + * Target Processor: Cortex-M cores + * + * -------------------------------------------------------------------- */ + +#include "arm_math.h" +#include "arm_nnfunctions.h" + +/** + * @ingroup groupNN + */ + +/** + * @addtogroup Softmax + * @{ + */ + + /** + * @brief Q7 softmax function + * @param[in] vec_in pointer to input vector + * @param[in] dim_vec input vector dimention + * @param[out] p_out pointer to output vector + * @return none. + * + * @details + * + * Here, instead of typical natural logarithm e based softmax, we use + * 2-based softmax here, i.e.,: + * + * y_i = 2^(x_i) / sum(2^x_j) + * + * The relative output will be different here. + * But mathematically, the gradient will be the same + * with a log(2) scaling factor. + * + */ + +void arm_softmax_q7(const q7_t * vec_in, const uint16_t dim_vec, q7_t * p_out) +{ + q31_t sum; + int16_t i; + uint8_t shift; + q15_t base; + base = -257; + + /* We first search for the maximum */ + for (i = 0; i < dim_vec; i++) + { + if (vec_in[i] > base) + { + base = vec_in[i]; + } + } + + /* + * So the base is set to max-8, meaning + * that we ignore really small values. + * anyway, they will be 0 after shrinking to q7_t. + */ + base = base - 8; + + sum = 0; + + for (i = 0; i < dim_vec; i++) + { + if (vec_in[i] > base) + { + shift = (uint8_t)__USAT(vec_in[i] - base, 5); + sum += 0x1 << shift; + } + } + + /* This is effectively (0x1 << 20) / sum */ + int output_base = 0x100000 / sum; + + /* + * Final confidence will be output_base >> ( 13 - (vec_in[i] - base) ) + * so 128 (0x1<<7) -> 100% confidence when sum = 0x1 << 8, output_base = 0x1 << 12 + * and vec_in[i]-base = 8 + */ + for (i = 0; i < dim_vec; i++) + { + if (vec_in[i] > base) + { + /* Here minimum value of 13+base-vec_in[i] will be 5 */ + shift = (uint8_t)__USAT(13+base-vec_in[i], 5); + p_out[i] = (q7_t) __SSAT((output_base >> shift), 8); + } else { + p_out[i] = 0; + } + } +} + +/** + * @} end of Softmax group + */ -- cgit