Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

AVX2 vpmadd GEMM move constants to functions to params #6802

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

AVX2 vpmadd GEMM move constants to functions to params #6802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 20 additions & 12 deletions src/amalgam/gen/avx2.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3573,9 +3573,11 @@ void xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avx2_madd_prfm(
const __m256i vinput_zero_point0 = _mm256_set1_epi32((int) quantization_params[0].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -3723,9 +3725,11 @@ void xnn_qd8_f16_qc4w_gemm_minmax_ukernel_4x8c8__avx2_madd_prfm(
const __m256i vinput_zero_point3 = _mm256_set1_epi32((int) quantization_params[3].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -5203,9 +5207,11 @@ void xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x8c8__avx2_madd_prfm(
const __m256i vinput_zero_point0 = _mm256_set1_epi32((int) quantization_params[0].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -5353,9 +5359,11 @@ void xnn_qd8_f32_qc4w_gemm_minmax_ukernel_4x8c8__avx2_madd_prfm(
const __m256i vinput_zero_point3 = _mm256_set1_epi32((int) quantization_params[3].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down
32 changes: 20 additions & 12 deletions src/amalgam/gen/avx256skx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,11 @@ void xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avx256skx_madd_prfm(
const __m256i vinput_zero_point0 = _mm256_set1_epi32((int) quantization_params[0].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -215,9 +217,11 @@ void xnn_qd8_f16_qc4w_gemm_minmax_ukernel_8x8c8__avx256skx_madd_prfm(
const __m256i vinput_zero_point7 = _mm256_set1_epi32((int) quantization_params[7].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -1263,9 +1267,11 @@ void xnn_qd8_f32_qc4w_gemm_minmax_ukernel_1x8c8__avx256skx_madd_prfm(
const __m256i vinput_zero_point0 = _mm256_set1_epi32((int) quantization_params[0].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -1430,9 +1436,11 @@ void xnn_qd8_f32_qc4w_gemm_minmax_ukernel_8x8c8__avx256skx_madd_prfm(
const __m256i vinput_zero_point7 = _mm256_set1_epi32((int) quantization_params[7].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask == (int8_t) 0x0F);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0x0F
assert(params->avxvnni.mask[0] == (int8_t) 0x0F);
const __m256i vzero_point = _mm256_load_si256((const __m256i*) params->avxvnni.zero_point); // 8
const __m256i vsixteen = _mm256_load_si256((const __m256i*) params->avxvnni.sixteen); // 16
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down
20 changes: 10 additions & 10 deletions src/amalgam/gen/avx256vnni.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ void xnn_qd8_f16_qc4w_gemm_minmax_ukernel_1x8c8__avx256vnni(
const __m256i vinput_zero_point0 = _mm256_set1_epi32((int) quantization_params[0].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0xF0
assert(params->avxvnni.mask == (int8_t) 0xF0);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0xF0
assert(params->avxvnni.mask[0] == (int8_t) 0xF0);
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -211,9 +211,9 @@ void xnn_qd8_f16_qc4w_gemm_minmax_ukernel_8x8c8__avx256vnni(
const __m256i vinput_zero_point7 = _mm256_set1_epi32((int) quantization_params[7].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_set1_epi8(params->avxvnni.mask); // 0xF0
assert(params->avxvnni.mask == (int8_t) 0xF0);
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
const __m256i vmask = _mm256_load_si256((const __m256i*) params->avxvnni.mask); // 0xF0
assert(params->avxvnni.mask[0] == (int8_t) 0xF0);
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -513,7 +513,7 @@ void xnn_qd8_f16_qc8w_gemm_minmax_ukernel_1x8c8__avx256vnni(
const __m256i vinput_zero_point0 = _mm256_set1_epi32((int) quantization_params[0].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -670,7 +670,7 @@ void xnn_qd8_f16_qc8w_gemm_minmax_ukernel_8x8c8__avx256vnni(
const __m256i vinput_zero_point7 = _mm256_set1_epi32((int) quantization_params[7].zero_point + 128);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
__m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point0);
Expand Down Expand Up @@ -959,7 +959,7 @@ void xnn_qd8_f16_qc8w_igemm_minmax_ukernel_1x8c8__avx256vnni(
const __m256 vinput_inv_scale = _mm256_set1_ps(quantization_params->inv_scale);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
const __m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point);
Expand Down Expand Up @@ -1112,7 +1112,7 @@ void xnn_qd8_f16_qc8w_igemm_minmax_ukernel_8x8c8__avx256vnni(
const __m256 vinput_inv_scale = _mm256_set1_ps(quantization_params->inv_scale);
const __m256 voutput_min = _mm256_set1_ps(params->avxvnni.min);
const __m256 voutput_max = _mm256_set1_ps(params->avxvnni.max);
const __m256i vsign_mask = _mm256_set1_epi8(params->avxvnni.sign_mask); // 0x80
const __m256i vsign_mask = _mm256_load_si256((const __m256i*) params->avxvnni.sign_mask); // 0x80
do {
const __m256i vksum01234567 = _mm256_load_si256(w);
const __m256i vsum0x01234567 = _mm256_mullo_epi32(vksum01234567, vinput_zero_point);
Expand Down
Loading
Loading