VECT: Support loop len control on EXTRACT_LAST vectorization
Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for powerpc and s390 with your suggestions: - machine_mode len_load_mode = get_len_load_store_mode - (loop_vinfo->vector_mode, true).require (); - machine_mode len_store_mode = get_len_load_store_mode - (loop_vinfo->vector_mode, false).require (); + machine_mode len_load_mode, len_store_mode; + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true) + .exists (&len_load_mode)) + return false; + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false) + .exists (&len_store_mode)) + return false; Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com> gcc/ChangeLog: * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check. (vectorizable_live_operation): Add live vectorization for length loop control. gcc/testsuite/ChangeLog: * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test. * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
This commit is contained in:
Juzhe-Zhong
Pan Li
parent
710d54f4c5
commit
c27f06260b
3 changed files with 138 additions and 20 deletions
34
gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
Normal file
34
gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
/* { dg-do compile } */
|
||||
/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
|
||||
|
||||
#include <stdint-gcc.h>
|
||||
|
||||
#define EXTRACT_LAST(TYPE) \
|
||||
TYPE __attribute__ ((noinline, noclone)) \
|
||||
test_##TYPE (TYPE *x, int n, TYPE value) \
|
||||
{ \
|
||||
TYPE last; \
|
||||
for (int j = 0; j < n; ++j) \
|
||||
{ \
|
||||
last = x[j]; \
|
||||
x[j] = last * value; \
|
||||
} \
|
||||
return last; \
|
||||
}
|
||||
|
||||
#define TEST_ALL(T) \
|
||||
T (int8_t) \
|
||||
T (int16_t) \
|
||||
T (int32_t) \
|
||||
T (int64_t) \
|
||||
T (uint8_t) \
|
||||
T (uint16_t) \
|
||||
T (uint32_t) \
|
||||
T (uint64_t) \
|
||||
T (_Float16) \
|
||||
T (float) \
|
||||
T (double)
|
||||
|
||||
TEST_ALL (EXTRACT_LAST)
|
||||
|
||||
/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
/* { dg-do run { target { riscv_vector } } } */
|
||||
/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
|
||||
|
||||
#include "live-1.c"
|
||||
|
||||
#define N 107
|
||||
#define OP 70
|
||||
|
||||
#define TEST_LOOP(TYPE) \
|
||||
{ \
|
||||
TYPE a[N]; \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
{ \
|
||||
a[i] = i * 2 + (i % 3); \
|
||||
asm volatile ("" ::: "memory"); \
|
||||
} \
|
||||
TYPE expected = a[N - 1]; \
|
||||
TYPE res = test_##TYPE (a, N, OP); \
|
||||
if (res != expected) \
|
||||
__builtin_abort (); \
|
||||
for (int i = 0; i < N; ++i) \
|
||||
{ \
|
||||
TYPE old = i * 2 + (i % 3); \
|
||||
if (a[i] != (TYPE) (old * (TYPE) OP)) \
|
||||
__builtin_abort (); \
|
||||
asm volatile ("" ::: "memory"); \
|
||||
} \
|
||||
}
|
||||
|
||||
int __attribute__ ((optimize (1)))
|
||||
main (void)
|
||||
{
|
||||
TEST_ALL (TEST_LOOP);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see
|
|||
#include "tree-pass.h"
|
||||
#include "ssa.h"
|
||||
#include "optabs-tree.h"
|
||||
#include "memmodel.h"
|
||||
#include "optabs.h"
|
||||
#include "diagnostic-core.h"
|
||||
#include "fold-const.h"
|
||||
#include "stor-layout.h"
|
||||
|
|
@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
|
|||
if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
|
||||
return false;
|
||||
|
||||
machine_mode len_load_mode = get_len_load_store_mode
|
||||
(loop_vinfo->vector_mode, true).require ();
|
||||
machine_mode len_store_mode = get_len_load_store_mode
|
||||
(loop_vinfo->vector_mode, false).require ();
|
||||
machine_mode len_load_mode, len_store_mode;
|
||||
if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
|
||||
.exists (&len_load_mode))
|
||||
return false;
|
||||
if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
|
||||
.exists (&len_store_mode))
|
||||
return false;
|
||||
|
||||
signed char partial_load_bias = internal_len_load_store_bias
|
||||
(IFN_LEN_LOAD, len_load_mode);
|
||||
|
|
@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
|
|||
/* No transformation required. */
|
||||
if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
|
||||
{
|
||||
if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
|
||||
OPTIMIZE_FOR_SPEED))
|
||||
{
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
|
||||
"can't operate on partial vectors "
|
||||
"because the target doesn't support extract "
|
||||
"last reduction.\n");
|
||||
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
|
||||
}
|
||||
else if (slp_node)
|
||||
if (slp_node)
|
||||
{
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
|
||||
|
|
@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
|
|||
else
|
||||
{
|
||||
gcc_assert (ncopies == 1 && !slp_node);
|
||||
vect_record_loop_mask (loop_vinfo,
|
||||
&LOOP_VINFO_MASKS (loop_vinfo),
|
||||
1, vectype, NULL);
|
||||
if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
|
||||
OPTIMIZE_FOR_SPEED))
|
||||
vect_record_loop_mask (loop_vinfo,
|
||||
&LOOP_VINFO_MASKS (loop_vinfo),
|
||||
1, vectype, NULL);
|
||||
else if (can_vec_extract_var_idx_p (
|
||||
TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
|
||||
vect_record_loop_len (loop_vinfo,
|
||||
&LOOP_VINFO_LENS (loop_vinfo),
|
||||
1, vectype, 1);
|
||||
else
|
||||
{
|
||||
if (dump_enabled_p ())
|
||||
dump_printf_loc (
|
||||
MSG_MISSED_OPTIMIZATION, vect_location,
|
||||
"can't operate on partial vectors "
|
||||
"because the target doesn't support extract "
|
||||
"last reduction.\n");
|
||||
LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
/* ??? Enable for loop costing as well. */
|
||||
|
|
@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
|
|||
gimple *vec_stmt;
|
||||
if (slp_node)
|
||||
{
|
||||
gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
|
||||
gcc_assert (!loop_vinfo
|
||||
|| (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
|
||||
&& !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
|
||||
|
||||
/* Get the correct slp vectorized stmt. */
|
||||
vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
|
||||
|
|
@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
|
|||
|
||||
gimple_seq stmts = NULL;
|
||||
tree new_tree;
|
||||
if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
|
||||
if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
|
||||
{
|
||||
/* Emit:
|
||||
|
||||
SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
|
||||
|
||||
where VEC_LHS is the vectorized live-out result and MASK is
|
||||
the loop mask for the final iteration. */
|
||||
gcc_assert (ncopies == 1 && !slp_node);
|
||||
gimple_seq tem = NULL;
|
||||
gimple_stmt_iterator gsi = gsi_last (tem);
|
||||
tree len
|
||||
= vect_get_loop_len (loop_vinfo, &gsi,
|
||||
&LOOP_VINFO_LENS (loop_vinfo),
|
||||
1, vectype, 0, 0);
|
||||
|
||||
/* BIAS - 1. */
|
||||
signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
|
||||
tree bias_minus_one
|
||||
= int_const_binop (MINUS_EXPR,
|
||||
build_int_cst (TREE_TYPE (len), biasval),
|
||||
build_one_cst (TREE_TYPE (len)));
|
||||
|
||||
/* LAST_INDEX = LEN + (BIAS - 1). */
|
||||
tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
|
||||
len, bias_minus_one);
|
||||
|
||||
/* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
|
||||
tree scalar_res
|
||||
= gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
|
||||
vec_lhs_phi, last_index);
|
||||
|
||||
/* Convert the extracted vector element to the scalar type. */
|
||||
new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
|
||||
}
|
||||
else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
|
||||
{
|
||||
/* Emit:
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue