Add a general mapping from internal fns to target insns
Several existing internal functions map directly to an instruction defined in target-insns.def. This patch makes it easier to define more such functions in future. This should help to reduce cut-&-paste, but more importantly, it allows the difference between optab functions and target-insns.def functions to be abstracted away; both are now treated as “directly-mapped”. gcc/ * internal-fn.def (DEF_INTERNAL_INSN_FN): New macro. (GOMP_SIMT_ENTER_ALLOC, GOMP_SIMT_EXIT, GOMP_SIMT_LANE) (GOMP_SIMT_LAST_LANE, GOMP_SIMT_ORDERED_PRED, GOMP_SIMT_VOTE_ANY) (GOMP_SIMT_XCHG_BFLY, GOMP_SIMT_XCHG_IDX): Use it. * internal-fn.h (direct_internal_fn_info::directly_mapped): New member variable. (direct_internal_fn_info::vectorizable): Reduce to 1 bit. (direct_internal_fn_p): Also return true for internal functions that map directly to instructions defined target-insns.def. (direct_internal_fn): Adjust comment accordingly. * internal-fn.cc (direct_insn, optab1, optab2, vectorizable_optab1) (vectorizable_optab2): New local macros. (not_direct): Initialize directly_mapped. (mask_load_direct, load_lanes_direct, mask_load_lanes_direct) (gather_load_direct, len_load_direct, mask_store_direct) (store_lanes_direct, mask_store_lanes_direct, vec_cond_mask_direct) (vec_cond_direct, scatter_store_direct, len_store_direct) (vec_set_direct, unary_direct, binary_direct, ternary_direct) (cond_unary_direct, cond_binary_direct, cond_ternary_direct) (while_direct, fold_extract_direct, fold_left_direct) (mask_fold_left_direct, check_ptrs_direct): Use the macros above. (expand_GOMP_SIMT_ENTER_ALLOC, expand_GOMP_SIMT_EXIT): Delete (expand_GOMP_SIMT_LANE, expand_GOMP_SIMT_LAST_LANE): Likewise; (expand_GOMP_SIMT_ORDERED_PRED, expand_GOMP_SIMT_VOTE_ANY): Likewise. (expand_GOMP_SIMT_XCHG_BFLY, expand_GOMP_SIMT_XCHG_IDX): Likewise. (direct_internal_fn_types): Handle functions that map to instructions defined in target-insns.def. (direct_internal_fn_types): Likewise. (direct_internal_fn_supported_p): Likewise. (internal_fn_expanders): Likewise.
This commit is contained in:
Richard Sandiford
parent
1d205dbac1
commit
f8baf4004e
3 changed files with 87 additions and 119 deletions
|
|
@ -105,37 +105,44 @@ init_internal_fns ()
|
|||
|
||||
/* Create static initializers for the information returned by
|
||||
direct_internal_fn. */
|
||||
#define not_direct { -2, -2, false }
|
||||
#define mask_load_direct { -1, 2, false }
|
||||
#define load_lanes_direct { -1, -1, false }
|
||||
#define mask_load_lanes_direct { -1, -1, false }
|
||||
#define gather_load_direct { 3, 1, false }
|
||||
#define len_load_direct { -1, -1, false }
|
||||
#define mask_store_direct { 3, 2, false }
|
||||
#define store_lanes_direct { 0, 0, false }
|
||||
#define mask_store_lanes_direct { 0, 0, false }
|
||||
#define vec_cond_mask_direct { 1, 0, false }
|
||||
#define vec_cond_direct { 2, 0, false }
|
||||
#define scatter_store_direct { 3, 1, false }
|
||||
#define len_store_direct { 3, 3, false }
|
||||
#define vec_set_direct { 3, 3, false }
|
||||
#define unary_direct { 0, 0, true }
|
||||
#define binary_direct { 0, 0, true }
|
||||
#define ternary_direct { 0, 0, true }
|
||||
#define cond_unary_direct { 1, 1, true }
|
||||
#define cond_binary_direct { 1, 1, true }
|
||||
#define cond_ternary_direct { 1, 1, true }
|
||||
#define while_direct { 0, 2, false }
|
||||
#define fold_extract_direct { 2, 2, false }
|
||||
#define fold_left_direct { 1, 1, false }
|
||||
#define mask_fold_left_direct { 1, 1, false }
|
||||
#define check_ptrs_direct { 0, 0, false }
|
||||
#define not_direct { -2, -2, false, false }
|
||||
#define direct_insn { -2, -2, true, false }
|
||||
#define optab1(TYPE0) { TYPE0, TYPE0, true, false }
|
||||
#define optab2(TYPE0, TYPE1) { TYPE0, TYPE1, true, false }
|
||||
#define vectorizable_optab1(TYPE0) { TYPE0, TYPE0, true, true }
|
||||
|
||||
#define mask_load_direct optab2 (-1, 2)
|
||||
#define load_lanes_direct optab1 (-1)
|
||||
#define mask_load_lanes_direct optab1 (-1)
|
||||
#define gather_load_direct optab2 (3, 1)
|
||||
#define len_load_direct optab1 (-1)
|
||||
#define mask_store_direct optab2 (3, 2)
|
||||
#define store_lanes_direct optab1 (0)
|
||||
#define mask_store_lanes_direct optab1 (0)
|
||||
#define vec_cond_mask_direct optab2 (1, 0)
|
||||
#define vec_cond_direct optab2 (2, 0)
|
||||
#define scatter_store_direct optab2 (3, 1)
|
||||
#define len_store_direct optab1 (3)
|
||||
#define vec_set_direct optab1 (3)
|
||||
#define unary_direct vectorizable_optab1 (0)
|
||||
#define binary_direct vectorizable_optab1 (0)
|
||||
#define ternary_direct vectorizable_optab1 (0)
|
||||
#define cond_unary_direct vectorizable_optab1 (1)
|
||||
#define cond_binary_direct vectorizable_optab1 (1)
|
||||
#define cond_ternary_direct vectorizable_optab1 (1)
|
||||
#define while_direct optab2 (0, 2)
|
||||
#define fold_extract_direct optab1 (2)
|
||||
#define fold_left_direct optab1 (1)
|
||||
#define mask_fold_left_direct optab1 (1)
|
||||
#define check_ptrs_direct optab1 (0)
|
||||
|
||||
const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
|
||||
#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
|
||||
#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
|
||||
#define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
|
||||
UNSIGNED_OPTAB, TYPE) TYPE##_direct,
|
||||
#define DEF_INTERNAL_INSN_FN(CODE, FLAGS, INSN, NOUTPUTS, NINPUTS) \
|
||||
direct_insn,
|
||||
#include "internal-fn.def"
|
||||
not_direct
|
||||
};
|
||||
|
|
@ -308,34 +315,6 @@ expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
|
|||
gcc_unreachable ();
|
||||
}
|
||||
|
||||
/* Allocate per-lane storage and begin non-uniform execution region. */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_enter ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_enter, 1, 2);
|
||||
}
|
||||
|
||||
/* Deallocate per-lane storage and leave non-uniform execution region. */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_exit ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_exit, 0, 1);
|
||||
}
|
||||
|
||||
/* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
|
||||
without SIMT execution this should be expanded in omp_device_lower pass. */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_lane ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_lane, 1, 0);
|
||||
}
|
||||
|
||||
/* This should get expanded in omp_device_lower pass. */
|
||||
|
||||
static void
|
||||
|
|
@ -344,55 +323,6 @@ expand_GOMP_SIMT_VF (internal_fn, gcall *)
|
|||
gcc_unreachable ();
|
||||
}
|
||||
|
||||
/* Lane index of the first SIMT lane that supplies a non-zero argument.
|
||||
This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
|
||||
lane that executed the last iteration for handling OpenMP lastprivate. */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_last_lane ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_last_lane, 1, 1);
|
||||
}
|
||||
|
||||
/* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_ordered ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_ordered, 1, 1);
|
||||
}
|
||||
|
||||
/* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
|
||||
any lane supplies a non-zero argument. */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_vote_any ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_vote_any, 1, 1);
|
||||
}
|
||||
|
||||
/* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
|
||||
is destination lane index XOR given offset. */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_xchg_bfly ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_xchg_bfly, 1, 2);
|
||||
}
|
||||
|
||||
/* Exchange between SIMT lanes according to given source lane index. */
|
||||
|
||||
static void
|
||||
expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
|
||||
{
|
||||
gcc_assert (targetm.have_omp_simt_xchg_idx ());
|
||||
expand_fn_using_insn (stmt, targetm.code_for_omp_simt_xchg_idx, 1, 2);
|
||||
}
|
||||
|
||||
/* This should get expanded in adjust_simduid_builtins. */
|
||||
|
||||
static void
|
||||
|
|
@ -3633,6 +3563,10 @@ tree_pair
|
|||
direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
|
||||
{
|
||||
const direct_internal_fn_info &info = direct_internal_fn (fn);
|
||||
if (info.type0 == -2)
|
||||
/* Functions created by DEF_INTERNAL_INSN_FN are not type-dependent. */
|
||||
return tree_pair {};
|
||||
|
||||
tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
|
||||
tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
|
||||
return tree_pair (type0, type1);
|
||||
|
|
@ -3646,6 +3580,10 @@ tree_pair
|
|||
direct_internal_fn_types (internal_fn fn, gcall *call)
|
||||
{
|
||||
const direct_internal_fn_info &info = direct_internal_fn (fn);
|
||||
if (info.type0 == -2)
|
||||
/* Functions created by DEF_INTERNAL_INSN_FN are not type-dependent. */
|
||||
return tree_pair {};
|
||||
|
||||
tree op0 = (info.type0 < 0
|
||||
? gimple_call_lhs (call)
|
||||
: gimple_call_arg (call, info.type0));
|
||||
|
|
@ -3790,6 +3728,8 @@ direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
|
|||
return direct_##TYPE##_optab_supported_p (which_optab, types, \
|
||||
opt_type); \
|
||||
}
|
||||
#define DEF_INTERNAL_INSN_FN(CODE, FLAGS, INSN, NOUTPUTS, NINPUTS) \
|
||||
case IFN_##CODE: return targetm.have_##INSN ();
|
||||
#include "internal-fn.def"
|
||||
|
||||
case IFN_LAST:
|
||||
|
|
@ -3941,6 +3881,14 @@ set_edom_supported_p (void)
|
|||
optab which_optab = direct_internal_fn_optab (fn, types); \
|
||||
expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
|
||||
}
|
||||
#define DEF_INTERNAL_INSN_FN(CODE, FLAGS, INSN, NOUTPUTS, NINPUTS) \
|
||||
static void \
|
||||
expand_##CODE (internal_fn, gcall *stmt) \
|
||||
{ \
|
||||
gcc_assert (targetm.have_##INSN ()); \
|
||||
expand_fn_using_insn (stmt, targetm.code_for_##INSN, \
|
||||
NOUTPUTS, NINPUTS); \
|
||||
}
|
||||
#include "internal-fn.def"
|
||||
|
||||
/* Routines to expand each internal function, indexed by function number.
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ along with GCC; see the file COPYING3. If not see
|
|||
UNSIGNED_OPTAB, TYPE)
|
||||
DEF_INTERNAL_FLT_FN (NAME, FLAGS, OPTAB, TYPE)
|
||||
DEF_INTERNAL_INT_FN (NAME, FLAGS, OPTAB, TYPE)
|
||||
DEF_INTERNAL_INSN_FN (NAME, FLAGS, INSN, NOUTPUTS, NINPUTS)
|
||||
|
||||
where NAME is the name of the function, FLAGS is a set of
|
||||
ECF_* flags and FNSPEC is a string describing functions fnspec.
|
||||
|
|
@ -82,6 +83,11 @@ along with GCC; see the file COPYING3. If not see
|
|||
says that the function extends the C-level BUILT_IN_<NAME>{,L,LL,IMAX}
|
||||
group of functions to any integral mode (including vector modes).
|
||||
|
||||
DEF_INTERNAL_INSN_FN defines an internal function that maps to target
|
||||
instruction INSN, which is one of those defined in target-insns.def.
|
||||
The instruction has NOUTPUTS output operands (either 0 or 1) and
|
||||
NINPUTS input operands.
|
||||
|
||||
Each entry must have a corresponding expander of the form:
|
||||
|
||||
void expand_NAME (gimple_call stmt)
|
||||
|
|
@ -120,6 +126,11 @@ along with GCC; see the file COPYING3. If not see
|
|||
DEF_INTERNAL_OPTAB_FN (NAME, FLAGS, OPTAB, TYPE)
|
||||
#endif
|
||||
|
||||
#ifndef DEF_INTERNAL_INSN_FN
|
||||
#define DEF_INTERNAL_INSN_FN(NAME, FLAGS, INSN, NOUTPUTS, NINPUTS) \
|
||||
DEF_INTERNAL_FN (NAME, FLAGS | ECF_LEAF, NULL)
|
||||
#endif
|
||||
|
||||
DEF_INTERNAL_OPTAB_FN (MASK_LOAD, ECF_PURE, maskload, mask_load)
|
||||
DEF_INTERNAL_OPTAB_FN (LOAD_LANES, ECF_CONST, vec_load_lanes, load_lanes)
|
||||
DEF_INTERNAL_OPTAB_FN (MASK_LOAD_LANES, ECF_PURE,
|
||||
|
|
@ -315,15 +326,21 @@ DEF_INTERNAL_INT_FN (POPCOUNT, ECF_CONST | ECF_NOTHROW, popcount, unary)
|
|||
|
||||
DEF_INTERNAL_FN (GOMP_USE_SIMT, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_ENTER, ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_ENTER_ALLOC, ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_EXIT, ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_LANE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_ENTER_ALLOC, ECF_NOTHROW, omp_simt_enter, 1, 2)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_EXIT, ECF_NOTHROW, omp_simt_exit, 0, 1)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_LANE, ECF_NOVOPS | ECF_NOTHROW,
|
||||
omp_simt_lane, 1, 0)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_VF, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_LAST_LANE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_ORDERED_PRED, ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_VOTE_ANY, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_XCHG_BFLY, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMT_XCHG_IDX, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_LAST_LANE, ECF_NOVOPS | ECF_NOTHROW,
|
||||
omp_simt_last_lane, 1, 1)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_ORDERED_PRED, ECF_NOTHROW,
|
||||
omp_simt_ordered, 1, 1)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_VOTE_ANY, ECF_NOVOPS | ECF_NOTHROW,
|
||||
omp_simt_vote_any, 1, 1)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_XCHG_BFLY, ECF_NOVOPS | ECF_NOTHROW,
|
||||
omp_simt_xchg_bfly, 1, 2)
|
||||
DEF_INTERNAL_INSN_FN (GOMP_SIMT_XCHG_IDX, ECF_NOVOPS | ECF_NOTHROW,
|
||||
omp_simt_xchg_idx, 1, 2)
|
||||
DEF_INTERNAL_FN (GOMP_SIMD_LANE, ECF_NOVOPS | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMD_VF, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
DEF_INTERNAL_FN (GOMP_SIMD_LAST_LANE, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
|
|
@ -433,6 +450,7 @@ DEF_INTERNAL_FN (SHUFFLEVECTOR, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
|
|||
/* <=> optimization. */
|
||||
DEF_INTERNAL_FN (SPACESHIP, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
|
||||
|
||||
#undef DEF_INTERNAL_INSN_FN
|
||||
#undef DEF_INTERNAL_INT_FN
|
||||
#undef DEF_INTERNAL_FLT_FN
|
||||
#undef DEF_INTERNAL_FLT_FLOATN_FN
|
||||
|
|
|
|||
|
|
@ -133,7 +133,8 @@ internal_fn_fnspec (enum internal_fn fn)
|
|||
return internal_fn_fnspec_array[(int) fn];
|
||||
}
|
||||
|
||||
/* Describes an internal function that maps directly to an optab. */
|
||||
/* Describes an internal function that maps directly to either an optab
|
||||
or an instruction defined in target-insns.def. */
|
||||
struct direct_internal_fn_info
|
||||
{
|
||||
/* optabs can be parameterized by one or two modes. These fields describe
|
||||
|
|
@ -144,24 +145,25 @@ struct direct_internal_fn_info
|
|||
function isn't directly mapped to an optab. */
|
||||
signed int type0 : 8;
|
||||
signed int type1 : 8;
|
||||
/* True if the function is directly mapped to either an optab or an
|
||||
instruction defined in target-insns.def. */
|
||||
unsigned int directly_mapped : 1;
|
||||
/* True if the function is pointwise, so that it can be vectorized by
|
||||
converting the return type and all argument types to vectors of the
|
||||
same number of elements. E.g. we can vectorize an IFN_SQRT on
|
||||
floats as an IFN_SQRT on vectors of N floats.
|
||||
|
||||
This only needs 1 bit, but occupies the full 16 to ensure a nice
|
||||
layout. */
|
||||
unsigned int vectorizable : 16;
|
||||
floats as an IFN_SQRT on vectors of N floats. */
|
||||
unsigned int vectorizable : 1;
|
||||
};
|
||||
|
||||
extern const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1];
|
||||
|
||||
/* Return true if FN is mapped directly to an optab. */
|
||||
/* Return true if FN is mapped directly to either an optab or an instruction
|
||||
defined in target-insns.def. */
|
||||
|
||||
inline bool
|
||||
direct_internal_fn_p (internal_fn fn)
|
||||
{
|
||||
return direct_internal_fn_array[fn].type0 >= -1;
|
||||
return direct_internal_fn_array[fn].directly_mapped;
|
||||
}
|
||||
|
||||
/* Return true if FN is a direct internal function that can be vectorized by
|
||||
|
|
@ -175,7 +177,7 @@ vectorizable_internal_fn_p (internal_fn fn)
|
|||
return direct_internal_fn_array[fn].vectorizable;
|
||||
}
|
||||
|
||||
/* Return optab information about internal function FN. Only meaningful
|
||||
/* Return information about internal function FN. Only meaningful
|
||||
if direct_internal_fn_p (FN). */
|
||||
|
||||
inline const direct_internal_fn_info &
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue