diff -r a1e347446159 -r 28ccaba883f4 genericopenlibs/liboil/src/motovec/vec_memcpy.s --- a/genericopenlibs/liboil/src/motovec/vec_memcpy.s Fri Sep 17 19:25:42 2010 +0530 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,876 +0,0 @@ -//------------------------------------------------------------------ -// file: vec_memcpy.S -// AltiVec enabled version of memcpy and bcopy -//------------------------------------------------------------------ - -//------------------------------------------------------------------ -// Copyright Motorola, Inc. 2003 -// ALL RIGHTS RESERVED -// -// You are hereby granted a copyright license to use, modify, and -// distribute the SOFTWARE so long as this entire notice is retained -// without alteration in any modified and/or redistributed versions, -// and that such modified versions are clearly identified as such. -// No licenses are granted by implication, estoppel or otherwise under -// any patents or trademarks of Motorola, Inc. -// -// The SOFTWARE is provided on an "AS IS" basis and without warranty. -// To the maximum extent permitted by applicable law, MOTOROLA DISCLAIMS -// ALL WARRANTIES WHETHER EXPRESS OR IMPLIED, INCLUDING IMPLIED -// WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR -// PURPOSE AND ANY WARRANTY AGAINST INFRINGEMENT WITH -// REGARD TO THE SOFTWARE (INCLUDING ANY MODIFIED VERSIONS -// THEREOF) AND ANY ACCOMPANYING WRITTEN MATERIALS. -// -// To the maximum extent permitted by applicable law, IN NO EVENT SHALL -// MOTOROLA BE LIABLE FOR ANY DAMAGES WHATSOEVER -// (INCLUDING WITHOUT LIMITATION, DAMAGES FOR LOSS OF -// BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS -// INFORMATION, OR OTHER PECUNIARY LOSS) ARISING OF THE USE OR -// INABILITY TO USE THE SOFTWARE. Motorola assumes no responsibility -// for the maintenance and support of the SOFTWARE. -//------------------------------------------------------------------ - -//------------------------------------------------------------------ -// extern void * memcpy(void *dst, const void *src, size_t len); -// Returns: -// void *dst -//------------------------------------------------------------------ - -//------------------------------------------------------------------ -// extern void * memmove( void *dst, const void *src, size_t len ); -// Copies len characters from src to dst and returns the value of -// dst. Works correctly for overlapping memory regions. -// - Harbison&Steele 4th ed (corrected as to return) -// Returns: -// void *dst -//------------------------------------------------------------------ - -//------------------------------------------------------------------ -// extern void * bcopy(const void *src, void *dst, size_t len); -// Returns: -// void *dst -//------------------------------------------------------------------ - -// memcpy and memmove are combined into one entry point here because of -// the similarity of operation and need to create fool-proof code. -// The following conditions determine what is "fool proof": -// -// if: then single entry: -// (DST-SRC)<0 && (SRC-DST)>=BC && BC>MIN_VEC will b to v_memcpy -// (DST-SRC)<0 && (SRC-DST)< BC && BC>MIN_VEC must b to v_memcpy -// (DST-SRC)<0 && BC0 && BC0 && (DST-SRC)< BC && BC>MIN_VEC must b to v_memmove -// (DST-SRC)>0 && (SRC-DST)>=BC && BC>MIN_VEC will b to v_memmove - -// If you call memmove (or vec_memmove) and |DST-SRC|>=BC, -// this code will branch to v_memcpy anyway for maximum performance. - -// Revision History: -// Rev 0.0 Original Chuck Corley 02/03/03 -// Can still add dst, 128B loop, and aligned option -// Rev 0.01 Fixed JY's seg-fault violation CJC 02/17/03 -// Rev 0.1 Added 128B loop and dst; cndtnlzd dcbz CJC 02/18/03 -// (Creating separate path for QW aligned didn't help much) -// Rev 0.11 Small code schdling; chngd dst for memmove CJC 02/23/03 -// Rev 0.20 Eliminated alternate entry and cleanup CJC 02/27/03 -// Rev 0.21 Inproved loop branch targets for v_mempcy CJC 03/01/03 -// Rev 0.22 Experimented with dst (sent to H.) CJC 03/02/03 -// Rev 0.23 Substituted dcba for dcbz (sent to JY) CJC 03/08/03 -// Rev 0.24 Use two dst streams CJC 03/12/03 -// Rev 0.25 Fix for all compilers, cleanup, and release with -// libmotovec.a rev 0.10 CJC 03/14/03 -// Rev 0.30 Fix for pre-empted destination (SNDF-DS) CJC 04/02/03 -// -// Between Rev 0.25 and 0.30 the code was revised to store elements of -// source at destination when first and/or last vector are less than 16 -// bytes. Areviewer at SNDF observed that loading the destination vector -// for merging exposed the "uninvolved" destination bytes to incoherency -// if an interrupt pre-empted this routine and modified the "uninvolved" -// destination vector(s) while held in register for merging. It seems -// like a low possibility but this revision is no longer subject to that -// possibility. (It is also slightly faster than Rev 0.25.) -// This is beta quality code; users are encouraged to make it faster. -// ASSUMPTIONS: -// Code is highly likely to be in the cache; data is not (streaming data) - -#define VRSV 256 // VRSAVE spr -// Don't use vectors for BC <= MIN_VEC. Works only if MIN_VEC >= 16 bytes. -#define MIN_VEC 16 -// Don't use Big_loop in v_memcpy for |dst-src|<= minimum overlap. -#define MIN_OVL 128 - -// Register useage -#define Rt r0 // r0 when used as a temporary register - -#define DST r3 // entering: dst pointer; exiting: same dst pointer - -#define SRC r4 // entering: src ptr; then end of src range index (SRC+BC) in memmove - -#define BC r5 // entering: Byte_Count - -#define PCS r6 // save for partial checksum entering - -#define DMS r7 // dst - src initially -#define BK r7 // BC - 1 +/- (n*16) - -// Codewarrior will put an unwelcome space as "lbzu r0,1(r7 )" -// if you don't put the comment right after the r7. CJC 030314 -#define SM1 r8// src -1 for byte-by-byte forwards initially -#define S r8 // src[28:31] -#define SMD r8 // src[0:27]-dst[0:27] -#define STR r8 // data stream touch block & stride info for Big_loop - -#define DM1 r9// dst -1 for byte-by-byte forwards initially -#define D r9 // dst[28:31] -#define DNX r9 // (dst+n*16)[28:31] -#define BL r9 // second byte_kount index pointer - -#define SBC r10// src + byte count initially then src[28:31] -#define BLK r10 // temporary data stream touch block & stride info -#define DR r10 // (dst+16)[0:27] -#define QW r10 // number of quad words (vectors) - -#define DBC r11// dst + byte count initially -#define BLL r11 // temporary data stream touch block & stride info -#define SBK r11 // (src+byte_count-1) -#define SBR r11 // (src+byte_count-1)[0:27] -#define DBK r11 // (dst+byte_count-1) then (dst+byte_count-1)[28:31] -#define BIG r11 // QW/8 or 128 byte loop count -#define SP8 r11 // SRC + n*128 (8 QWs) for data streaming after first call - -#define RSV r12 // storage for VRSAVE register if used - -#define VS0 v0 // src vector for permuting - -#define VS1 v1 // src vector for permuting - -#define VP3 v2 // d - s permute register - -#define VPS0 v3 // permuted source vector to store - -#define VPS1 v4 // 2nd permuted source vector to store - -#define VPS2 v5 // additional permuted src in Big loop - -#define VS2 v6 // src vector for permuting -#define VPS3 v6 // additional permuted src in Big loop - -#define VS3 v7 // additional src load in Big loop -#define VPS4 v7 // additional permuted src in Big loop - -#define VS4 v8 // additional src load in Big loop -#define VPS5 v8 // additional permuted src in Big loop - -#define VS5 v9 // additional src load in Big loop -#define VPS6 v9 // additional permuted src in Big loop - -#define VS6 v10 // additional src load in Big loop -#define VPS7 v10 // additional permuted src in Big loop - -#define VS7 v11 // additional src load in Big loop - -// Conditionalize the use of dcba. It will help if the data is -// not in cache and hurt if it is. Generally, except for small -// benchmarks repeated many times, we assume data is not in cache -// (data streaming) and using dcbz is a performance boost. -#ifndef NO_DCBA -#if defined(__GNUC__) || defined(__MWERKS__) || defined(_DIAB_TOOL) - // gcc and codewarrior and diab don't assemble dcba -#define DCBK .long 0x7c033dec -// dcba r3,r7 or dcba DST,BK -#define DCBL .long 0x7c034dec -// dcba r3,r9 or dcba DST,BL -#else -#ifdef __ghs__ -.macro DCBK -.long 0x7c033dec -.endm -.macro DCBL -.long 0x7c034dec -.endm -#else -#define DCBK dcba DST,BK -#define DCBL dcba DST,BL -#endif // __ghs__ -#endif // __GNUC__ or __MWERKS__ -#else -#define DCBK nop -#define DCBL nop -#endif // NO_DCBA - -// Conditionalize the use of dst (data stream touch). It will help -// if the data is not in cache and hurt if it is (though not as badly -// as dcbz). Generally, except for small benchmarks repeated many times, -// we assume data is not in cache (data streaming) and using dst is a -// performance boost. -#ifndef NO_DST -#define STRM_B dst SBC,BLL,0 -#define STRM_F dst SRC,BLK,0 -#define STRM_1 dst SP8,STR,1 - -#else -#define STRM_B nop -#define STRM_F nop -#define STRM_1 nop -#endif - -// Condition register use -// cr0[0:2] = (dst-src==0)? return: ((dst-src>0)? copy_bkwd, copy_fwd;); -// then cr0[0:2] = (dst[28:31]-src[28:31]<0)? "shifting left", "shifting right"; -// cr1[0,2] = (BC == 0)? 1 : 0; (nothing to move) -// then cr1[2] = (DST[28:31] == 0)? 1 : 0; (D0 left justified) -// then cr1[2] = ((DBK = DST+BC-1)[28:31] = 0xF)? 1 : 0; (DN right justified) -// cr5[0,2] = (|DST-SRC|<=MIN_OVL)?1:0; (Overlap too small for Big loop?) -// cr6[1,2] = (DST-SRC>=BC)?1:0; (Okay for v_memmove to copy forward?) -// then cr6[2] = (QW == 0)? 1 : 0; (Any full vectors to move?) -// then cr6[1] = (QW > 4)? 1 : 0; (>4 vectors to move?) -// then cr6[3] = (third store[27] == 1)? 1: 0; (cache line alignment) -// then cr6[3] = (last store[27] == 1)? 1: 0; (last store odd?) -// cr7[2] = (BC>MIN_VEC)?1:0; (BC big enough to warrant vectors) -// then cr7[0:3] = (DST+16)[0:27]-DST (How many bytes (iff <16) in first vector?) -// then cr7[1] = (QW > 14)? 1 : 0; (>14 vectors to move?) -// then cr7[0:3] = (DST+BC)[0:27] (How many bytes (iff <16) in last vector?) - - .text -#ifdef __MWERKS__ - .align 32 -#else - .align 5 -#endif - -#ifdef LIBMOTOVEC - .globl memmove -memmove: - nop // IU1 Compilers forget first label - .globl memcpy -memcpy: -#else - .globl vec_memmove -vec_memmove: - nop // IU1 Only way I know to preserve both labels - .globl _vec_memcpy -_vec_memcpy: -#endif - subf. DMS,SRC,DST // IU1 Compute dst-src difference - cmpi cr1,0,BC,0 // IU1 Eliminate zero byte count moves - cmpi cr7,0,BC,MIN_VEC // IU1 Check for minimum byte count - - addi SM1,SRC,-1 // IU1 Pre-bias and duplicate src for fwd - addi DM1,DST,-1 // IU1 Pre-bias and duplicate destination - add SBC,SRC,BC // IU1 Pre-bias and duplicate src for bkwd - beqlr // return if DST = SRC - - add DBC,DST,BC // IU1 Pre-bias and duplicate destination - subf Rt,DST,SRC // IU1 Form |DST-SRC| if DST-SRC<0 - beqlr cr1 // return if BC = 0 - - bgt Cpy_bkwd // b if DST-SRC>0 (have to copy backward) - cmpi cr5,0,Rt,MIN_OVL // IU1 (|DST-SRC|>128)?1:0; for v_memcpy - bgt cr7,v_memcpy // b if BC>MIN_VEC (okay to copy vectors fwd) - -// Copy byte-by-byte forwards if DST-SRC<0 and BC<=MIN_VEC - mtctr BC // i=BC; do ...;i--; while (i>0) -Byte_cpy_fwd: - lbzu Rt,1(SM1) // LSU * ++(DST-1) = * ++(SRC-1) - stbu Rt,1(DM1) // LSU - bdnz Byte_cpy_fwd - - blr - nop // IU1 Improve next label as branch target -Cpy_bkwd: - cmpi cr5,0,DMS,MIN_OVL // IU1 ((DST-SRC)>128)?1:0; for v_memcpy - cmp cr6,0,DMS,BC // IU1 cr6[1,2]=(DST-SRC>=BC)?1:0; - bgt cr7,v_memmove // b if BC>MIN_VEC (copy vectors bkwd) -// Copy byte-by-byte backwards if DST-SRC>0 and BC<=MIN_VEC - mtctr BC // i=BC; do ...;i--; while (i>0) -Byte_cpy_bwd: - lbzu Rt,-1(SBC) // LSU * --(DST+BC) = * --(SRC+BC) - stbu Rt,-1(DBC) // LSU Store it - bdnz Byte_cpy_bwd - blr - -#ifdef __MWERKS__ - .align 16 -#else - .align 4 -#endif - -v_memmove: -// Byte count < MIN_VEC bytes will have been copied by scalar code above, -// so this will not deal with small block moves < MIN_VEC. - -// For systems using VRSAVE, define VRSAVE=1 when compiling. For systems -// that don't, make sure VRSAVE is undefined. -#ifdef VRSAVE - mfspr RSV,VRSV // IU2 Get current VRSAVE contents -#endif - rlwinm S,SRC,0,28,31 // IU1 Save src address bits s[28:31] - rlwinm D,DST,0,28,31 // IU1 D = dst[28:31] - bge cr6,MC_entry // b to v_memcpy if DST-SRC>=BC (fwd copy OK) - -#ifdef VRSAVE - oris Rt,RSV,0xfff0 // IU1 Or in registers used by this routine -#endif - lis BLL,0x010c // IU1 Stream 12 blocks of 16 bytes - subf. SMD,D,S // IU1 if S-D<0 essentially shifting right - -#ifdef VRSAVE - mtspr VRSV,Rt // IU2 Save in VRSAVE before first vec op -#endif - lvsr VP3,0,DMS // LSU Permute vector for dst - src shft right - ori BLL,BLL,0xffe0 // IU1 Stream stride -32B - - STRM_B // LSU Start data stream at SRC+BC - addi SBK,SBC,-1 // IU1 Address of last src byte - bgt Rt_shft // Bytes from upper vector = (s-d>0)?s-d:16+s-d; - addi SMD,SMD,16 // IU1 Save 16-(d-s) -Rt_shft: - - rlwinm SBR,SBK,0,0,27 // IU1 (SRC+BC-1)[0:27] - addi BK,BC,-1 // IU1 Initialize byte index - - subf Rt,SBR,SBC // IU1 How many bytes in first source? - add DBK,DST,BK // IU1 Address of last dst byte - addi DR,DST,16 // IU1 Address of second dst vector - - subf. SMD,Rt,SMD // IU1 if bytes in 1st src>Bytes in 1st permute - rlwinm Rt,DBK,0,28,31 // IU1 (DST+BC-1)[28:31] - rlwinm DR,DR,0,0,27 // IU1 (DST+16)[0:27] - -// If there are more useful bytes in the upper vector of a permute pair than we -// will get in the first permute, the first loaded vector needs to be in the -// lower half of the permute pair. The upper half is a don't care then. - blt Get_bytes_rt // b if shifting left (D-S>=0) - - lvx VS1,SRC,BK // LSU Get SN load started -// Comments numbering source and destination assume single path through the -// code executing each instruction once. For vec_memmove, an example would -// be the call memmove(BASE+0x0F, BASE+0x2F, 82). N = 6 in that case. - addi SRC,SRC,-16 // IU1 Decrement src base (to keep BK useful) - -Get_bytes_rt: // Come here to get VS0 & Don't care what VS1 is - lvx VS0,SRC,BK // LSU Get SN-1 (SN if D-S<0) in lower vector - subf QW,DR,DBK // IU1 Bytes of full vectors to move (-16) - cmpi cr7,0,Rt,0xF // IU1 Is Dn right justified? - - cmpi cr1,0,D,0 // IU1 Is D0 left justified? - rlwinm QW,QW,28,4,31 // IU1 Quad words remaining - add Rt,DST,BC // IU1 Refresh the value of DST+BC - - cmpi cr6,0,QW,0 // IU1 Any full vectors to move? - vperm VPS0,VS0,VS1,VP3 // VPU Align SN-1 and SN to DN - vor VS1,VS0,VS0 // VIU1 Move lower vector to upper - beq cr7,Rt_just // b if DN is right justified - - mtcrf 0x01,Rt // IU2 Put final vector byte count in cr7 - rlwinm DBK,DBK,0,0,27 // IU1 Address of first byte of final vector - li D,0 // IU1 Initialize an index pointer - bnl cr7,Only_1W_bkwd // b if there was only one or zero words to store - - stvewx VPS0,DBK,D // LSU store word 1 of two or three - addi D,D,4 // IU1 increment index - - stvewx VPS0,DBK,D // LSU store word 2 of two or three - addi D,D,4 // IU1 increment index -Only_1W_bkwd: - bng cr7,Only_2W_bkwd // b if there were only two or zero words to store - - stvewx VPS0,DBK,D // LSU store word 3 of three if necessary - addi D,D,4 // IU1 increment index -Only_2W_bkwd: - bne cr7,Only_B_bkwd // b if there are no half words to store - - stvehx VPS0,DBK,D // LSU store one halfword if necessary - addi D,D,2 // IU1 increment index -Only_B_bkwd: - bns cr7,All_done_bkwd // b if there are no bytes to store - - stvebx VPS0,DBK,D // LSU store one byte if necessary - b All_done_bkwd - -Rt_just: - stvx VPS0,DST,BK // LSU Store 16 bytes at DN -All_done_bkwd: - addi BK,BK,-16 // IU1 Decrement destination byte count - - ble cr6,Last_load // b if no Quad words to do - mtctr QW // IU2 for (i=0;i<=QW;i++)-execution serializng - cmpi cr6,0,QW,4 // IU1 Check QW>4 -QW_loop: - lvx VS0,SRC,BK // LSU Get SN-2 (or SN-1 if ADJ==0) - - vperm VPS0,VS0,VS1,VP3 // VPU Align SN-2 and SN-1 to DN-1 - vor VS1,VS0,VS0 // VIU1 Move lower vector to upper - - stvx VPS0,DST,BK // LSU Store 16 bytes at DN-1 - addi BK,BK,-16 // IU1 Decrement byte kount - bdnzf 25,QW_loop // b if 4 or less quad words to do - - add DNX,DST,BK // IU1 address of next store (DST+BC-1-16) - bgt cr6,GT_4QW // b if >4 quad words left - -Last_load: // if D-S>=0, next load will be from same address as last - blt No_ld_bkwd // b if shifting right (S-D>=0) - addi SRC,SRC,16 // IU1 recorrect source if it was decremented -No_ld_bkwd: - lvx VS0,0,SRC // LSU Get last source SN-6 (guaranteed S0) -// Current 16 bytes is the last; we're done. - dss 0 // Data stream stop - vperm VPS0,VS0,VS1,VP3 // VPU Align SN-6 and SN-5 to DN-6 - subfic D,DST,16 // IU1 How many bytes in first destination? - beq cr1,Lt_just // b if last destination is left justified - - mtcrf 0x01,D // IU2 Put byte count remaining in cr7 - li D,0 // IU1 Initialize index pointer - bns cr7,No_B_bkwd // b if only even number of bytes to store - - stvebx VPS0,DST,D // LSU store first byte at DST+0 - addi D,D,1 // IU1 increment index -No_B_bkwd: - bne cr7,No_H_bkwd // b if only words to store - stvehx VPS0,DST,D // LSU store halfword at DST+0/1 - addi D,D,2 // IU1 increment index - -No_H_bkwd: - bng cr7,No_W1_bkwd // b if exactly zero or two words to store - stvewx VPS0,DST,D // LSU store word 1 of one or three - addi D,D,4 // IU1 increment index - -No_W1_bkwd: - bnl cr7,No_W2_bkwd // b if there was only one word to store - stvewx VPS0,DST,D // LSU store word 1 of two or 2 of three - addi D,D,4 // IU1 increment index - - stvewx VPS0,DST,D // LSU store word 2 of two or 3 of three - b No_W2_bkwd - -Lt_just: - stvx VPS0,0,DST // LSU Store 16 bytes at final dst addr D0 -No_W2_bkwd: -#ifdef VRSAVE - mtspr VRSV,RSV // IU1 Restore VRSAVE -#endif - blr // Return destination address from entry - -GT_4QW: // Do once if next store is to even half of cache line, else twice - - lvx VS0,SRC,BK // LSU Get SN-3 (or SN-2) - mtcrf 0x02,DNX // IU2 cr6[3]=((DST+BC-1)[27]==1)?1:0; - - vperm VPS0,VS0,VS1,VP3 // VPU Align SN-3 and SN-2 to Dn-2 - vor VS1,VS0,VS0 // VIU1 Move lower vector to upper - addi DNX,DNX,-16 // IU1 Prepare to update cr6 next loop - - stvx VPS0,DST,BK // LSU Store 16 bytes at DN-2 - vor VS3,VS0,VS0 // VIU Make a copy of lower vector - addi BK,BK,-16 // IU1 Decrement byte count by 16 - bdnzt 27,GT_4QW // b if next store is to upper (odd) half of CL -// At this point next store will be to even address. - - lis STR,0x102 // IU1 Stream 2 blocks of 16 bytes - mtcrf 0x02,DST // IU2 cr6[3]=(DST[27]==1)?1:0; (DST odd?) - addi BL,BK,-16 // IU1 Create an alternate byte count - 16 - - ori STR,STR,0xffe0 // IU1 Stream stride -32B - addi SP8,SRC,-64 // IU1 Starting address for data stream touch - bso cr6,B32_bkwd // b if DST[27] == 1; i.e, final store is odd - - bdnz B32_bkwd // decrement counter for last odd QW store -B32_bkwd: // Should be at least 2 stores remaining and next 2 are cache aligned - lvx VS2,SRC,BK // LSU Get SN-4 (or SN-3) - addi SP8,SP8,-32 // IU1 Next starting address for data stream touch - - lvx VS1,SRC,BL // LSU Get SN-5 (or SN-4) - vperm VPS0,VS2,VS3,VP3 // VPU Align SN-4 and SN-3 to DN-3 - - STRM_1 // LSU Stream 64 byte blocks ahead of loads - - DCBL // LSU allocate next cache line - - vperm VPS1,VS1,VS2,VP3 // VPU Align SN-5 and SN-4 to DN-4 - vor VS3,VS1,VS1 // VIU1 Move SN-5 to SN-3 - - stvx VPS0,DST,BK // LSU Store 16 bytes at DN-3 - addi BK,BL,-16 // IU1 Decrement byte count - bdz Nxt_loc_bkwd // always decrement and branch to next instr - -Nxt_loc_bkwd: - stvx VPS1,DST,BL // LSU Store 16 bytes at DN-4 - addi BL,BK,-16 // IU1 Decrement alternate byte count - bdnz B32_bkwd // b if there are at least two more QWs to do - - bns cr6,One_odd_QW // b if there was one more odd QW to store - b Last_load - -// Come here with two more loads and two stores to do -One_odd_QW: - lvx VS1,SRC,BK // LSU Get SN-6 (or SN-5) - - vperm VPS1,VS1,VS3,VP3 // VPU Align SN-6 and SN-5 to DN-5 - - stvx VPS1,DST,BK // LSU Store 16 bytes at DN-5 - - b Last_load - -// End of memmove in AltiVec - -#ifdef __MWERKS__ - .align 16 -#else - .align 4 -#endif -v_memcpy: -// Byte count < MIN_VEC bytes will have been copied by scalar code above, -// so this will not deal with small block moves < MIN_VEC. - -#ifdef VRSAVE - mfspr RSV,VRSV // IU2 Get current VRSAVE contents -#endif - rlwinm S,SRC,0,28,31 // IU1 Save src address bits s[28:31] - rlwinm D,DST,0,28,31 // IU1 D = dst[28:31] - -MC_entry: // enter here from memmove if DST-SRC>=BC; this should be faster -#ifdef VRSAVE - oris Rt,RSV,0xfff0 // IU1 Or in registers used by this routine -#endif - lis BLK,0x010c // IU1 Stream 12 blocks of 16 bytes - - subf. S,S,D // IU1 if D-S<0 essentially shifting left - -#ifdef VRSAVE - mtspr VRSV,Rt // IU2 Save in VRSAVE before first vec op -#endif - lvsr VP3,0,DMS // LSU Permute vector for dst - src shft right - ori BLK,BLK,32 // IU1 Stream stride 32B - - STRM_F // LSU Start data stream 0 at SRC - addi DR,DST,16 // IU1 Address of second dst vector - addi DBK,DBC,-1 // IU1 Address of last dst byte - -// If D-S<0 we are "kinda" shifting left with the right shift permute vector -// loaded to VP3 and we need both S0 and S1 to permute. If D-S>=0 then the -// first loaded vector needs to be in the upper half of the permute pair and -// the lower half is a don't care then. - bge Ld_bytes_rt // b if shifting right (D-S>=0) - - lvx VS0,0,SRC // LSU Get S0 load started -// Comments numbering source and destination assume single path through the -// code executing each instruction once. For vec_memcpy, an example would -// be the call memcpy(BASE+0x1E, BASE+0x1F, 259). N = 16 in that case. - addi SRC,SRC,16 // IU1 Increment src base (to keep BK useful) - -Ld_bytes_rt: // Come here to get VS1 & Don't care what VS0 is - lvx VS1,0,SRC // LSU Get S1 (or S0 if D-S>=0) in upper vector - rlwinm DR,DR,0,0,27 // IU1 (DST+16)[0:27] - cmpi cr1,0,D,0 // IU1 Is D0 left justified? - - subf Rt,DST,DR // IU1 How many bytes in first destination? - subf QW,DR,DBK // IU1 Bytes of full vectors to move (-16) - li BK,0 // IU1 Initialize byte kount index - - mtcrf 0x01,Rt // IU2 Put bytes in 1st dst in cr7 - rlwinm QW,QW,28,4,31 // IU1 Quad words remaining - vperm VPS0,VS0,VS1,VP3 // VPU Align S0 and S1 to D0 - - vor VS0,VS1,VS1 // VIU1 Move upper vector to lower - beq cr1,Left_just // b if D0 is left justified - - bns cr7,No_B_fwd // b if only even number of bytes to store - - stvebx VPS0,DST,BK // LSU store first byte at DST+0 - addi BK,BK,1 // IU1 increment index -No_B_fwd: - bne cr7,No_H_fwd // b if only words to store - - stvehx VPS0,DST,BK // LSU store halfword at DST+0/1 - addi BK,BK,2 // IU1 increment index -No_H_fwd: - bng cr7,No_W1_fwd // b if exactly zero or two words to store - - stvewx VPS0,DST,BK // LSU store word 1 of one or three - addi BK,BK,4 // IU1 increment index - -No_W1_fwd: - bnl cr7,No_W2_fwd // b if there was only one word to store - stvewx VPS0,DST,BK // LSU store word 1 of two or 2 of three - addi BK,BK,4 // IU1 increment index - - stvewx VPS0,DST,BK // LSU store word 2 of two or 3 of three - b No_W2_fwd - -Left_just: - stvx VPS0,0,DST // LSU Store 16 bytes at D0 -No_W2_fwd: - rlwinm Rt,DBK,0,28,31 // IU1 (DBK = DST+BC-1)[28:31] - cmpi cr6,0,QW,0 // IU1 Any full vectors to move? - - li BK,16 // IU1 Re-initialize byte kount index - cmpi cr1,0,Rt,0xF // IU1 Is DN right justified? - cmpi cr7,0,QW,14 // IU1 Check QW>14 - ble cr6,Last_ld_fwd // b if no Quad words to do - - mtctr QW // IU2 for (i=0;i<=QW;i++) - cmpi cr6,0,QW,4 // IU1 Check QW>4 -QW_fwd_loop: - lvx VS1,SRC,BK // LSU Get S2 (or S1) - - vperm VPS0,VS0,VS1,VP3 // VPU Align S1 and S2 to D1 - vor VS0,VS1,VS1 // VIU1 Move upper vector to lower - - stvx VPS0,DST,BK // LSU Store 16 bytes at D1(+n*16 where n<4) - addi BK,BK,16 // IU1 Increment byte kount index - bdnzf 25,QW_fwd_loop // b if 4 or less quad words to do - - add DNX,DST,BK // IU1 address of next store (DST+32 if QW>4) - addi QW,QW,-1 // IU1 One more QW stored by now - bgt cr6,GT_4QW_fwd // b if >4 quad words left - -Last_ld_fwd: // Next 16 bytes is the last; we're done. - add DBC,DST,BC // IU1 Recompute address of last dst byte + 1 - add SBC,SRC,BC // IU1 Recompute address of last src byte + 1 - bge No_ld_fwd // b if shifting right (D-S>=0) - - addi SBC,SBC,-16 // IU1 if D-S>=0 we didn't add 16 to src -No_ld_fwd: - mtcrf 0x01,DBC // IU2 Put final vector byte count in cr7 - addi DBK,DBC,-1 // IU1 Recompute address of last dst byte - addi Rt,SBC,-1 // IU1 Recompute address of last src byte - -// If D-S<0 we have already loaded all the source vectors. -// If D-S>=0 then the first loaded vector went to the upper half of the permute -// pair and we need one more vector. (This may be a duplicate.) - - lvx VS1,0,Rt // LSU Get last source S14 (guaranteed SN) - -#ifndef NO_DST - dss 0 // Data stream 0 stop - - dss 1 // Data stream 1 stop -#endif - vperm VPS0,VS0,VS1,VP3 // VPU Align S13 and S14 to D14 - beq cr1,Rt_just_fwd // b if last destination is right justified - - rlwinm DBK,DBK,0,0,27 // IU1 Round to QW addr of last byte - li D,0 // IU1 Initialize index pointer - bnl cr7,Only_1W_fwd // b if there was only one or zero words to store - - stvewx VPS0,DBK,D // LSU store word 1 of two or three - addi D,D,4 // IU1 increment index - - stvewx VPS0,DBK,D // LSU store word 2 of two or three - addi D,D,4 // IU1 increment index -Only_1W_fwd: - bng cr7,Only_2W_fwd // b if there were only two or zero words to store - - stvewx VPS0,DBK,D // LSU store word 3 of three if necessary - addi D,D,4 // IU1 increment index -Only_2W_fwd: - bne cr7,Only_B_fwd // b if there are no half words to store - - stvehx VPS0,DBK,D // LSU store one halfword if necessary - addi D,D,2 // IU1 increment index -Only_B_fwd: - bns cr7,All_done_fwd // b if there are no bytes to store - - stvebx VPS0,DBK,D // LSU store one byte if necessary - b All_done_fwd - -Rt_just_fwd: - - stvx VPS0,DST,BK // LSU Store 16 bytes at D14 -All_done_fwd: -#ifdef VRSAVE - mtspr VRSV,RSV // IU1 Restore VRSAVE -#endif - blr // Return destination address from entry -#ifdef __MWERKS__ - .align 16 -#else - .align 4 -#endif -GT_4QW_fwd: // Do once if nxt st is to odd half of cache line, else twice - - lvx VS1,SRC,BK // LSU Get S3 (or S2) - addi QW,QW,-1 // IU1 Keeping track of QWs stored - mtcrf 0x02,DNX // IU2 cr6[3]=((DST+32)[27]==1)?1:0; - - addi DNX,DNX,16 // IU1 Update cr6 for next loop - addi Rt,QW,-2 // IU1 Insure at least 2 QW left after big loop - - vperm VPS0,VS0,VS1,VP3 // VPU Align S2 and S3 to D2 - vor VS0,VS1,VS1 // VIU1 Move upper vector to lower - - stvx VPS0,DST,BK // LSU Store 16 bytes at D2 - addi BK,BK,16 // IU1 Increment byte count by 16 - bdnzf 27,GT_4QW_fwd // b if next store is to lower (even) half of CL -// At this point next store will be to even address. - - mtcrf 0x02,DBK // IU2 cr6[3]=((last store)[27]==1)?1:0; (odd?) - lis STR,0x104 // IU1 Stream 4 blocks of 16 bytes - addi BL,BK,16 // IU1 Create an alternate byte kount + 32 - - ori STR,STR,32 // IU1 Stream stride 32B -#ifndef NO_BIG_LOOP - rlwinm BIG,Rt,29,3,31 // IU1 QW/8 big loops to do - - rlwinm Rt,Rt,0,0,28 // IU1 How many QWs will be done in big loop - bgt cr7,Big_loop // b if QW > 14 -#endif -No_big_loop: -// We need the ctr register to reflect an even byte count before entering -// the next block - faster to decrement than to reload. - - addi SP8,SRC,256 // IU1 Starting address for data stream touch - xoris STR,STR,0x6 // IU1 Reset stream to 2 blocks of 16 bytes - bns cr6,B32_fwd // b if DST[27] == 0; i.e, final store is even - - bdnz B32_fwd // decrement counter for last QW store odd - -B32_fwd: // Should be at least 2 stores remaining and next 2 are cache aligned - lvx VS1,SRC,BK // LSU Get S12 - addi SP8,SP8,32 // IU1 Next starting address for data stream touch - - lvx VS2,SRC,BL // LSU Get S13 - vperm VPS1,VS0,VS1,VP3 // VPU Align S11 and S12 to D11 - - STRM_1 // LSU Stream 64 byte blocks ahead of loads - - DCBK // LSU then Kill instead of RWITM - - vperm VPS0,VS1,VS2,VP3 // VPU Align S12 and S13 to D12 - vor VS0,VS2,VS2 // VIU1 Move S13 to S11 - - stvx VPS1,DST,BK // LSU Store 16 bytes at D11 - addi BK,BL,16 // IU1 Increment byte count - bdz Nxt_loc_fwd // always decrement and branch to next instr - -Nxt_loc_fwd: - stvx VPS0,DST,BL // LSU Store 16 bytes at D12 - addi BL,BK,16 // IU1 Increment alternate byte count - bdnz B32_fwd // b if there are at least two more QWs to do - - bso cr6,One_even_QW // b if there is one even and one odd QW to store - b Last_ld_fwd // b if last store is to even address - -// Come here with two more loads and two stores to do -One_even_QW: - lvx VS1,SRC,BK // LSU Get S14 (or S13 if if D-S>=0) - - vperm VPS0,VS0,VS1,VP3 // VPU Align S13 and S14 to D13 - vor VS0,VS1,VS1 // VIU1 Move upper vector to lower - - stvx VPS0,DST,BK // LSU Store 16 bytes at D13 - addi BK,BK,16 // IU1 Increment byte count - - b Last_ld_fwd - -#ifdef __MWERKS__ - .align 16 -#else - .align 4 -#endif -Big_loop: - subf QW,Rt,QW // IU1 Should be 2-7 QWs left after big loop - blt cr5,No_big_loop // b back if |DST-SRC|<128; Big_loop won't work. - mtctr BIG // IU2 loop for as many 128B loops as possible - addi SP8,SRC,256 // IU1 Starting address for data stream touch - -Loop_of_128B: // Come here with QW>=10 and next store even; VS0 last load - lvx VS1,SRC,BK // LSU Get S4 (or S3 if D-S>=0) - addi BL,BK,32 // IU1 Increment Byte_Kount+16 by 32 - addi SP8,SP8,128 // IU1 increment address for data stream touch - - lvx VS3,SRC,BL // LSU Get S6 (or S5) - addi BL,BL,32 // IU1 Increment Byte_Kount+48 by 32 - - lvx VS5,SRC,BL // LSU Get S8 (or S7) - addi BL,BL,32 // IU1 Increment Byte_Kount+80 by 32 - - lvx VS7,SRC,BL // LSU Get S10 (or S9) - addi BL,BK,16 // IU1 Increment Byte_Kount+16 by 16 - - lvx VS2,SRC,BL // LSU Get S5 (or S4) - addi BL,BL,32 // IU1 Increment Byte_Kount+32 by 32 - - lvx VS4,SRC,BL // LSU Get S7 (or S6) - addi BL,BL,32 // IU1 Increment Byte_Kount+64 by 32 - - lvx VS6,SRC,BL // LSU Get S9 (or S8) - addi BL,BL,32 // IU1 Increment Byte_Kount+96 by 32 - vperm VPS0,VS0,VS1,VP3 // VPU - - lvx VS0,SRC,BL // LSU Get S11 (or S10) - vperm VPS1,VS1,VS2,VP3 // VPU - - STRM_1 // LSU Stream 4 32B blocks, stride 32B - - DCBK // LSU then Kill instead of RWITM - - stvx VPS0,DST,BK // LSU Store D3 - addi BK,BK,16 // IU1 Increment Byte_Kount+16 by 16 - vperm VPS2,VS2,VS3,VP3 // VPU - - stvx VPS1,DST,BK // LSU Store D4 - addi BK,BK,16 // IU1 Increment Byte_Kount+32 by 16 - vperm VPS3,VS3,VS4,VP3 // VPU - - DCBK // LSU then Kill instead of RWITM - - stvx VPS2,DST,BK // LSU Store D5 - addi BK,BK,16 // IU1 Increment Byte_Kount+48 by 16 - vperm VPS4,VS4,VS5,VP3 // VPU - - stvx VPS3,DST,BK // LSU Store D6 - addi BK,BK,16 // IU1 Increment Byte_Kount+64 by 16 - vperm VPS5,VS5,VS6,VP3 // VPU - - DCBK // LSU then Kill instead of RWITM - - stvx VPS4,DST,BK // LSU Store D7 - addi BK,BK,16 // IU1 Increment Byte_Kount+80 by 16 - vperm VPS6,VS6,VS7,VP3 // VPU - - stvx VPS5,DST,BK // LSU Store D8 - addi BK,BK,16 // IU1 Increment Byte_Kount+96 by 16 - vperm VPS7,VS7,VS0,VP3 // VPU - - DCBK // LSU then Kill instead of RWITM - - stvx VPS6,DST,BK // LSU Store D9 - addi BK,BK,16 // IU1 Increment Byte_Kount+112 by 16 - - stvx VPS7,DST,BK // LSU Store D10 - addi BK,BK,16 // IU1 Increment Byte_Kount+128 by 16 - bdnz Loop_of_128B // b if ctr > 0 (QW/8 still > 0) - - mtctr QW // IU1 Restore QW remaining to counter - addi BL,BK,16 // IU1 Create an alternate byte kount + 16 - bns cr6,B32_fwd // b if DST[27] == 0; i.e, final store is even - - bdnz B32_fwd // b and decrement counter for last QW store odd - // One of the above branches should have taken - -// End of memcpy in AltiVec - -// bcopy works like memcpy, but the source and destination operands are reversed. -// Following will just reverse the operands and branch to memcpy. - -#ifdef LIBMOTOVEC - .globl bcopy -bcopy: -#else - .globl vec_bcopy -vec_bcopy: -#endif - mr Rt,DST // temp storage for what is really source address (r3) - mr DST,SRC // swap destination address to r3 to match memcpy dst - mr SRC,Rt // Complete swap of destination and source for memcpy -#ifdef LIBMOTOVEC - b memcpy // b to memcpy with correct args in r3 and r4 -#else - b _vec_memcpy // b to vec_memcpy with correct args in r3 and r4 -#endif -// End of bcopy in AltiVec