Projects
Mega:23.03
gcc
_service:tar_scm:0079-loop-vect-Transfer-arrays...
Sign Up
Log In
Username
Password
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File _service:tar_scm:0079-loop-vect-Transfer-arrays-using-registers-between-lo.patch of Package gcc
From 013544d0b477647c8835a8806c75e7b09155b8ed Mon Sep 17 00:00:00 2001 From: benniaobufeijiushiji <linda7@huawei.com> Date: Mon, 8 Aug 2022 09:13:53 +0800 Subject: [PATCH 31/35] [loop-vect] Transfer arrays using registers between loops For vectorized stores in loop, if all succeed loops immediately use the data, transfer data using registers instead of load store to prevent overhead from memory access. --- gcc/testsuite/gcc.dg/vect/vect-perm-1.c | 45 ++++++ gcc/tree-vect-stmts.c | 181 ++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 gcc/testsuite/gcc.dg/vect/vect-perm-1.c diff --git a/gcc/testsuite/gcc.dg/vect/vect-perm-1.c b/gcc/testsuite/gcc.dg/vect/vect-perm-1.c new file mode 100644 index 000000000..d8b29fbd5 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-perm-1.c @@ -0,0 +1,45 @@ +/* { dg-do compile { target { aarch64*-*-linux* } } } */ +/* { dg-options "-O3 -fdump-tree-vect-all-details -save-temps" } */ + +#include <stdio.h> +#include <stdlib.h> + +static unsigned inline abs2 (unsigned a) +{ + unsigned s = ((a>>15)&0x10001)*0xffff; + return (a+s)^s; +} + +int foo (unsigned *a00, unsigned *a11, unsigned *a22, unsigned *a33) +{ + unsigned tmp[4][4]; + unsigned a0, a1, a2, a3; + int sum = 0; + for (int i = 0; i < 4; i++) + { + int t0 = a00[i] + a11[i]; + int t1 = a00[i] - a11[i]; + int t2 = a22[i] + a33[i]; + int t3 = a22[i] - a33[i]; + tmp[i][0] = t0 + t2; + tmp[i][2] = t0 - t2; + tmp[i][1] = t1 + t3; + tmp[i][3] = t1 - t3; + } + for (int i = 0; i < 4; i++) + { + int t0 = tmp[0][i] + tmp[1][i]; + int t1 = tmp[0][i] - tmp[1][i]; + int t2 = tmp[2][i] + tmp[3][i]; + int t3 = tmp[2][i] - tmp[3][i]; + a0 = t0 + t2; + a2 = t0 - t2; + a1 = t1 + t3; + a3 = t1 - t3; + sum += abs2 (a0) + abs2 (a1) + abs2 (a2) + abs2 (a3); + } + return (((unsigned short) sum) + ((unsigned) sum >>16)) >> 1; +} + +/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" } } */ +/* { dg-final { scan-tree-dump-times "VEC_PERM_EXPR" 16 "vect" } } */ diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 2c2197022..98b233718 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -2276,6 +2276,173 @@ vector_vector_composition_type (tree vtype, poly_uint64 nelts, tree *ptype) return NULL_TREE; } +/* Check succeedor BB, BB without load is regarded as empty BB. Ignore empty + BB in DFS. */ + +static unsigned +mem_refs_in_bb (basic_block bb, vec<gimple *> &stmts) +{ + unsigned num = 0; + for (gimple_stmt_iterator gsi = gsi_start_bb (bb); + !gsi_end_p (gsi); gsi_next (&gsi)) + { + gimple *stmt = gsi_stmt (gsi); + if (is_gimple_debug (stmt)) + continue; + if (is_gimple_assign (stmt) && gimple_has_mem_ops (stmt) + && !gimple_has_volatile_ops (stmt)) + { + if (gimple_assign_rhs_code (stmt) == MEM_REF + || gimple_assign_rhs_code (stmt) == ARRAY_REF) + { + stmts.safe_push (stmt); + num++; + } + else if (TREE_CODE (gimple_get_lhs (stmt)) == MEM_REF + || TREE_CODE (gimple_get_lhs (stmt)) == ARRAY_REF) + num++; + } + } + return num; +} + +static bool +check_same_base (vec<data_reference_p> *datarefs, data_reference_p dr) +{ + for (unsigned ui = 0; ui < datarefs->length (); ui++) + { + tree op1 = TREE_OPERAND (DR_BASE_OBJECT (dr), 0); + tree op2 = TREE_OPERAND (DR_BASE_OBJECT ((*datarefs)[ui]), 0); + if (TREE_CODE (op1) != TREE_CODE (op2)) + continue; + if (TREE_CODE (op1) == ADDR_EXPR) + { + op1 = TREE_OPERAND (op1, 0); + op2 = TREE_OPERAND (op2, 0); + } + enum tree_code code = TREE_CODE (op1); + switch (code) + { + case VAR_DECL: + if (DECL_NAME (op1) == DECL_NAME (op2) + && DR_IS_READ ((*datarefs)[ui])) + return true; + break; + case SSA_NAME: + if (SSA_NAME_VERSION (op1) == SSA_NAME_VERSION (op2) + && DR_IS_READ ((*datarefs)[ui])) + return true; + break; + default: + break; + } + } + return false; +} + +/* Iterate all load STMTS, if staisfying same base vectorized stmt, then return, + Otherwise, set false to SUCCESS. */ + +static void +check_vec_use (loop_vec_info loop_vinfo, vec<gimple *> &stmts, + stmt_vec_info stmt_info, bool &success) +{ + if (stmt_info == NULL) + { + success = false; + return; + } + if (DR_IS_READ (stmt_info->dr_aux.dr)) + { + success = false; + return; + } + unsigned ui = 0; + gimple *candidate = NULL; + FOR_EACH_VEC_ELT (stmts, ui, candidate) + { + if (TREE_CODE (TREE_TYPE (gimple_get_lhs (candidate))) != VECTOR_TYPE) + continue; + + if (candidate->bb != candidate->bb->loop_father->header) + { + success = false; + return; + } + auto_vec<data_reference_p> datarefs; + tree res = find_data_references_in_bb (candidate->bb->loop_father, + candidate->bb, &datarefs); + if (res == chrec_dont_know) + { + success = false; + return; + } + if (check_same_base (&datarefs, stmt_info->dr_aux.dr)) + return; + } + success = false; +} + +/* Deep first search from present BB. If succeedor has load STMTS, + stop further searching. */ + +static void +dfs_check_bb (loop_vec_info loop_vinfo, basic_block bb, stmt_vec_info stmt_info, + bool &success, vec<basic_block> &visited_bbs) +{ + if (bb == cfun->cfg->x_exit_block_ptr) + { + success = false; + return; + } + if (!success || visited_bbs.contains (bb) || bb == loop_vinfo->loop->latch) + return; + + visited_bbs.safe_push (bb); + auto_vec<gimple *> stmts; + unsigned num = mem_refs_in_bb (bb, stmts); + /* Empty BB. */ + if (num == 0) + { + edge e; + edge_iterator ei; + FOR_EACH_EDGE (e, ei, bb->succs) + { + dfs_check_bb (loop_vinfo, e->dest, stmt_info, success, visited_bbs); + if (!success) + return; + } + return; + } + /* Non-empty BB. */ + check_vec_use (loop_vinfo, stmts, stmt_info, success); +} + +/* For grouped store, if all succeedors of present BB have vectorized load + from same base of store. If so, set memory_access_type using + VMAT_CONTIGUOUS_PERMUTE instead of VMAT_LOAD_STORE_LANES. */ + +static bool +conti_perm (stmt_vec_info stmt_vinfo, loop_vec_info loop_vinfo) +{ + gimple *stmt = stmt_vinfo->stmt; + if (gimple_code (stmt) != GIMPLE_ASSIGN) + return false; + + if (DR_IS_READ (stmt_vinfo->dr_aux.dr)) + return false; + + basic_block bb = stmt->bb; + bool success = true; + auto_vec<basic_block> visited_bbs; + visited_bbs.safe_push (bb); + edge e; + edge_iterator ei; + FOR_EACH_EDGE (e, ei, bb->succs) + dfs_check_bb (loop_vinfo, e->dest, stmt_vinfo, success, visited_bbs); + return success; +} + /* A subroutine of get_load_store_type, with a subset of the same arguments. Handle the case where STMT_INFO is part of a grouped load or store. @@ -2434,6 +2601,20 @@ get_group_load_store_type (stmt_vec_info stmt_info, tree vectype, bool slp, *memory_access_type = VMAT_CONTIGUOUS_PERMUTE; overrun_p = would_overrun_p; } + + if (*memory_access_type == VMAT_LOAD_STORE_LANES + && TREE_CODE (loop_vinfo->num_iters) == INTEGER_CST + && maybe_eq (tree_to_shwi (loop_vinfo->num_iters), + loop_vinfo->vectorization_factor) + && conti_perm (stmt_info, loop_vinfo) + && (vls_type == VLS_LOAD + ? vect_grouped_load_supported (vectype, single_element_p, + group_size) + : vect_grouped_store_supported (vectype, group_size))) + { + *memory_access_type = VMAT_CONTIGUOUS_PERMUTE; + overrun_p = would_overrun_p; + } } /* As a last resort, trying using a gather load or scatter store. -- 2.27.0.windows.1
Locations
Projects
Search
Status Monitor
Help
Open Build Service
OBS Manuals
API Documentation
OBS Portal
Reporting a Bug
Contact
Mailing List
Forums
Chat (IRC)
Twitter
Open Build Service (OBS)
is an
openSUSE project
.
浙ICP备2022010568号-2