--- a/genericopenlibs/liboil/src/motovec/vec_memcpy.s Fri Sep 17 19:25:42 2010 +0530
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,876 +0,0 @@
-//------------------------------------------------------------------
-// file: vec_memcpy.S
-// AltiVec enabled version of memcpy and bcopy
-//------------------------------------------------------------------
-
-//------------------------------------------------------------------
-// Copyright Motorola, Inc. 2003
-// ALL RIGHTS RESERVED
-//
-// You are hereby granted a copyright license to use, modify, and
-// distribute the SOFTWARE so long as this entire notice is retained
-// without alteration in any modified and/or redistributed versions,
-// and that such modified versions are clearly identified as such.
-// No licenses are granted by implication, estoppel or otherwise under
-// any patents or trademarks of Motorola, Inc.
-//
-// The SOFTWARE is provided on an "AS IS" basis and without warranty.
-// To the maximum extent permitted by applicable law, MOTOROLA DISCLAIMS
-// ALL WARRANTIES WHETHER EXPRESS OR IMPLIED, INCLUDING IMPLIED
-// WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR
-// PURPOSE AND ANY WARRANTY AGAINST INFRINGEMENT WITH
-// REGARD TO THE SOFTWARE (INCLUDING ANY MODIFIED VERSIONS
-// THEREOF) AND ANY ACCOMPANYING WRITTEN MATERIALS.
-//
-// To the maximum extent permitted by applicable law, IN NO EVENT SHALL
-// MOTOROLA BE LIABLE FOR ANY DAMAGES WHATSOEVER
-// (INCLUDING WITHOUT LIMITATION, DAMAGES FOR LOSS OF
-// BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS
-// INFORMATION, OR OTHER PECUNIARY LOSS) ARISING OF THE USE OR
-// INABILITY TO USE THE SOFTWARE. Motorola assumes no responsibility
-// for the maintenance and support of the SOFTWARE.
-//------------------------------------------------------------------
-
-//------------------------------------------------------------------
-// extern void * memcpy(void *dst, const void *src, size_t len);
-// Returns:
-// void *dst
-//------------------------------------------------------------------
-
-//------------------------------------------------------------------
-// extern void * memmove( void *dst, const void *src, size_t len );
-// Copies len characters from src to dst and returns the value of
-// dst. Works correctly for overlapping memory regions.
-// - Harbison&Steele 4th ed (corrected as to return)
-// Returns:
-// void *dst
-//------------------------------------------------------------------
-
-//------------------------------------------------------------------
-// extern void * bcopy(const void *src, void *dst, size_t len);
-// Returns:
-// void *dst
-//------------------------------------------------------------------
-
-// memcpy and memmove are combined into one entry point here because of
-// the similarity of operation and need to create fool-proof code.
-// The following conditions determine what is "fool proof":
-//
-// if: then single entry:
-// (DST-SRC)<0 && (SRC-DST)>=BC && BC>MIN_VEC will b to v_memcpy
-// (DST-SRC)<0 && (SRC-DST)< BC && BC>MIN_VEC must b to v_memcpy
-// (DST-SRC)<0 && BC<MIN_VEC copy fwd byte-by-byte
-// (DST-SRC)==0 || BC==0 will just return
-// (DST-SRC)>0 && BC<MIN_VEC copy bkwd byte-by-byte
-// (DST-SRC)>0 && (DST-SRC)< BC && BC>MIN_VEC must b to v_memmove
-// (DST-SRC)>0 && (SRC-DST)>=BC && BC>MIN_VEC will b to v_memmove
-
-// If you call memmove (or vec_memmove) and |DST-SRC|>=BC,
-// this code will branch to v_memcpy anyway for maximum performance.
-
-// Revision History:
-// Rev 0.0 Original Chuck Corley 02/03/03
-// Can still add dst, 128B loop, and aligned option
-// Rev 0.01 Fixed JY's seg-fault violation CJC 02/17/03
-// Rev 0.1 Added 128B loop and dst; cndtnlzd dcbz CJC 02/18/03
-// (Creating separate path for QW aligned didn't help much)
-// Rev 0.11 Small code schdling; chngd dst for memmove CJC 02/23/03
-// Rev 0.20 Eliminated alternate entry and cleanup CJC 02/27/03
-// Rev 0.21 Inproved loop branch targets for v_mempcy CJC 03/01/03
-// Rev 0.22 Experimented with dst (sent to H.) CJC 03/02/03
-// Rev 0.23 Substituted dcba for dcbz (sent to JY) CJC 03/08/03
-// Rev 0.24 Use two dst streams CJC 03/12/03
-// Rev 0.25 Fix for all compilers, cleanup, and release with
-// libmotovec.a rev 0.10 CJC 03/14/03
-// Rev 0.30 Fix for pre-empted destination (SNDF-DS) CJC 04/02/03
-//
-// Between Rev 0.25 and 0.30 the code was revised to store elements of
-// source at destination when first and/or last vector are less than 16
-// bytes. Areviewer at SNDF observed that loading the destination vector
-// for merging exposed the "uninvolved" destination bytes to incoherency
-// if an interrupt pre-empted this routine and modified the "uninvolved"
-// destination vector(s) while held in register for merging. It seems
-// like a low possibility but this revision is no longer subject to that
-// possibility. (It is also slightly faster than Rev 0.25.)
-// This is beta quality code; users are encouraged to make it faster.
-// ASSUMPTIONS:
-// Code is highly likely to be in the cache; data is not (streaming data)
-
-#define VRSV 256 // VRSAVE spr
-// Don't use vectors for BC <= MIN_VEC. Works only if MIN_VEC >= 16 bytes.
-#define MIN_VEC 16
-// Don't use Big_loop in v_memcpy for |dst-src|<= minimum overlap.
-#define MIN_OVL 128
-
-// Register useage
-#define Rt r0 // r0 when used as a temporary register
-
-#define DST r3 // entering: dst pointer; exiting: same dst pointer
-
-#define SRC r4 // entering: src ptr; then end of src range index (SRC+BC) in memmove
-
-#define BC r5 // entering: Byte_Count
-
-#define PCS r6 // save for partial checksum entering
-
-#define DMS r7 // dst - src initially
-#define BK r7 // BC - 1 +/- (n*16)
-
-// Codewarrior will put an unwelcome space as "lbzu r0,1(r7 )"
-// if you don't put the comment right after the r7. CJC 030314
-#define SM1 r8// src -1 for byte-by-byte forwards initially
-#define S r8 // src[28:31]
-#define SMD r8 // src[0:27]-dst[0:27]
-#define STR r8 // data stream touch block & stride info for Big_loop
-
-#define DM1 r9// dst -1 for byte-by-byte forwards initially
-#define D r9 // dst[28:31]
-#define DNX r9 // (dst+n*16)[28:31]
-#define BL r9 // second byte_kount index pointer
-
-#define SBC r10// src + byte count initially then src[28:31]
-#define BLK r10 // temporary data stream touch block & stride info
-#define DR r10 // (dst+16)[0:27]
-#define QW r10 // number of quad words (vectors)
-
-#define DBC r11// dst + byte count initially
-#define BLL r11 // temporary data stream touch block & stride info
-#define SBK r11 // (src+byte_count-1)
-#define SBR r11 // (src+byte_count-1)[0:27]
-#define DBK r11 // (dst+byte_count-1) then (dst+byte_count-1)[28:31]
-#define BIG r11 // QW/8 or 128 byte loop count
-#define SP8 r11 // SRC + n*128 (8 QWs) for data streaming after first call
-
-#define RSV r12 // storage for VRSAVE register if used
-
-#define VS0 v0 // src vector for permuting
-
-#define VS1 v1 // src vector for permuting
-
-#define VP3 v2 // d - s permute register
-
-#define VPS0 v3 // permuted source vector to store
-
-#define VPS1 v4 // 2nd permuted source vector to store
-
-#define VPS2 v5 // additional permuted src in Big loop
-
-#define VS2 v6 // src vector for permuting
-#define VPS3 v6 // additional permuted src in Big loop
-
-#define VS3 v7 // additional src load in Big loop
-#define VPS4 v7 // additional permuted src in Big loop
-
-#define VS4 v8 // additional src load in Big loop
-#define VPS5 v8 // additional permuted src in Big loop
-
-#define VS5 v9 // additional src load in Big loop
-#define VPS6 v9 // additional permuted src in Big loop
-
-#define VS6 v10 // additional src load in Big loop
-#define VPS7 v10 // additional permuted src in Big loop
-
-#define VS7 v11 // additional src load in Big loop
-
-// Conditionalize the use of dcba. It will help if the data is
-// not in cache and hurt if it is. Generally, except for small
-// benchmarks repeated many times, we assume data is not in cache
-// (data streaming) and using dcbz is a performance boost.
-#ifndef NO_DCBA
-#if defined(__GNUC__) || defined(__MWERKS__) || defined(_DIAB_TOOL)
- // gcc and codewarrior and diab don't assemble dcba
-#define DCBK .long 0x7c033dec
-// dcba r3,r7 or dcba DST,BK
-#define DCBL .long 0x7c034dec
-// dcba r3,r9 or dcba DST,BL
-#else
-#ifdef __ghs__
-.macro DCBK
-.long 0x7c033dec
-.endm
-.macro DCBL
-.long 0x7c034dec
-.endm
-#else
-#define DCBK dcba DST,BK
-#define DCBL dcba DST,BL
-#endif // __ghs__
-#endif // __GNUC__ or __MWERKS__
-#else
-#define DCBK nop
-#define DCBL nop
-#endif // NO_DCBA
-
-// Conditionalize the use of dst (data stream touch). It will help
-// if the data is not in cache and hurt if it is (though not as badly
-// as dcbz). Generally, except for small benchmarks repeated many times,
-// we assume data is not in cache (data streaming) and using dst is a
-// performance boost.
-#ifndef NO_DST
-#define STRM_B dst SBC,BLL,0
-#define STRM_F dst SRC,BLK,0
-#define STRM_1 dst SP8,STR,1
-
-#else
-#define STRM_B nop
-#define STRM_F nop
-#define STRM_1 nop
-#endif
-
-// Condition register use
-// cr0[0:2] = (dst-src==0)? return: ((dst-src>0)? copy_bkwd, copy_fwd;);
-// then cr0[0:2] = (dst[28:31]-src[28:31]<0)? "shifting left", "shifting right";
-// cr1[0,2] = (BC == 0)? 1 : 0; (nothing to move)
-// then cr1[2] = (DST[28:31] == 0)? 1 : 0; (D0 left justified)
-// then cr1[2] = ((DBK = DST+BC-1)[28:31] = 0xF)? 1 : 0; (DN right justified)
-// cr5[0,2] = (|DST-SRC|<=MIN_OVL)?1:0; (Overlap too small for Big loop?)
-// cr6[1,2] = (DST-SRC>=BC)?1:0; (Okay for v_memmove to copy forward?)
-// then cr6[2] = (QW == 0)? 1 : 0; (Any full vectors to move?)
-// then cr6[1] = (QW > 4)? 1 : 0; (>4 vectors to move?)
-// then cr6[3] = (third store[27] == 1)? 1: 0; (cache line alignment)
-// then cr6[3] = (last store[27] == 1)? 1: 0; (last store odd?)
-// cr7[2] = (BC>MIN_VEC)?1:0; (BC big enough to warrant vectors)
-// then cr7[0:3] = (DST+16)[0:27]-DST (How many bytes (iff <16) in first vector?)
-// then cr7[1] = (QW > 14)? 1 : 0; (>14 vectors to move?)
-// then cr7[0:3] = (DST+BC)[0:27] (How many bytes (iff <16) in last vector?)
-
- .text
-#ifdef __MWERKS__
- .align 32
-#else
- .align 5
-#endif
-
-#ifdef LIBMOTOVEC
- .globl memmove
-memmove:
- nop // IU1 Compilers forget first label
- .globl memcpy
-memcpy:
-#else
- .globl vec_memmove
-vec_memmove:
- nop // IU1 Only way I know to preserve both labels
- .globl _vec_memcpy
-_vec_memcpy:
-#endif
- subf. DMS,SRC,DST // IU1 Compute dst-src difference
- cmpi cr1,0,BC,0 // IU1 Eliminate zero byte count moves
- cmpi cr7,0,BC,MIN_VEC // IU1 Check for minimum byte count
-
- addi SM1,SRC,-1 // IU1 Pre-bias and duplicate src for fwd
- addi DM1,DST,-1 // IU1 Pre-bias and duplicate destination
- add SBC,SRC,BC // IU1 Pre-bias and duplicate src for bkwd
- beqlr // return if DST = SRC
-
- add DBC,DST,BC // IU1 Pre-bias and duplicate destination
- subf Rt,DST,SRC // IU1 Form |DST-SRC| if DST-SRC<0
- beqlr cr1 // return if BC = 0
-
- bgt Cpy_bkwd // b if DST-SRC>0 (have to copy backward)
- cmpi cr5,0,Rt,MIN_OVL // IU1 (|DST-SRC|>128)?1:0; for v_memcpy
- bgt cr7,v_memcpy // b if BC>MIN_VEC (okay to copy vectors fwd)
-
-// Copy byte-by-byte forwards if DST-SRC<0 and BC<=MIN_VEC
- mtctr BC // i=BC; do ...;i--; while (i>0)
-Byte_cpy_fwd:
- lbzu Rt,1(SM1) // LSU * ++(DST-1) = * ++(SRC-1)
- stbu Rt,1(DM1) // LSU
- bdnz Byte_cpy_fwd
-
- blr
- nop // IU1 Improve next label as branch target
-Cpy_bkwd:
- cmpi cr5,0,DMS,MIN_OVL // IU1 ((DST-SRC)>128)?1:0; for v_memcpy
- cmp cr6,0,DMS,BC // IU1 cr6[1,2]=(DST-SRC>=BC)?1:0;
- bgt cr7,v_memmove // b if BC>MIN_VEC (copy vectors bkwd)
-// Copy byte-by-byte backwards if DST-SRC>0 and BC<=MIN_VEC
- mtctr BC // i=BC; do ...;i--; while (i>0)
-Byte_cpy_bwd:
- lbzu Rt,-1(SBC) // LSU * --(DST+BC) = * --(SRC+BC)
- stbu Rt,-1(DBC) // LSU Store it
- bdnz Byte_cpy_bwd
- blr
-
-#ifdef __MWERKS__
- .align 16
-#else
- .align 4
-#endif
-
-v_memmove:
-// Byte count < MIN_VEC bytes will have been copied by scalar code above,
-// so this will not deal with small block moves < MIN_VEC.
-
-// For systems using VRSAVE, define VRSAVE=1 when compiling. For systems
-// that don't, make sure VRSAVE is undefined.
-#ifdef VRSAVE
- mfspr RSV,VRSV // IU2 Get current VRSAVE contents
-#endif
- rlwinm S,SRC,0,28,31 // IU1 Save src address bits s[28:31]
- rlwinm D,DST,0,28,31 // IU1 D = dst[28:31]
- bge cr6,MC_entry // b to v_memcpy if DST-SRC>=BC (fwd copy OK)
-
-#ifdef VRSAVE
- oris Rt,RSV,0xfff0 // IU1 Or in registers used by this routine
-#endif
- lis BLL,0x010c // IU1 Stream 12 blocks of 16 bytes
- subf. SMD,D,S // IU1 if S-D<0 essentially shifting right
-
-#ifdef VRSAVE
- mtspr VRSV,Rt // IU2 Save in VRSAVE before first vec op
-#endif
- lvsr VP3,0,DMS // LSU Permute vector for dst - src shft right
- ori BLL,BLL,0xffe0 // IU1 Stream stride -32B
-
- STRM_B // LSU Start data stream at SRC+BC
- addi SBK,SBC,-1 // IU1 Address of last src byte
- bgt Rt_shft // Bytes from upper vector = (s-d>0)?s-d:16+s-d;
- addi SMD,SMD,16 // IU1 Save 16-(d-s)
-Rt_shft:
-
- rlwinm SBR,SBK,0,0,27 // IU1 (SRC+BC-1)[0:27]
- addi BK,BC,-1 // IU1 Initialize byte index
-
- subf Rt,SBR,SBC // IU1 How many bytes in first source?
- add DBK,DST,BK // IU1 Address of last dst byte
- addi DR,DST,16 // IU1 Address of second dst vector
-
- subf. SMD,Rt,SMD // IU1 if bytes in 1st src>Bytes in 1st permute
- rlwinm Rt,DBK,0,28,31 // IU1 (DST+BC-1)[28:31]
- rlwinm DR,DR,0,0,27 // IU1 (DST+16)[0:27]
-
-// If there are more useful bytes in the upper vector of a permute pair than we
-// will get in the first permute, the first loaded vector needs to be in the
-// lower half of the permute pair. The upper half is a don't care then.
- blt Get_bytes_rt // b if shifting left (D-S>=0)
-
- lvx VS1,SRC,BK // LSU Get SN load started
-// Comments numbering source and destination assume single path through the
-// code executing each instruction once. For vec_memmove, an example would
-// be the call memmove(BASE+0x0F, BASE+0x2F, 82). N = 6 in that case.
- addi SRC,SRC,-16 // IU1 Decrement src base (to keep BK useful)
-
-Get_bytes_rt: // Come here to get VS0 & Don't care what VS1 is
- lvx VS0,SRC,BK // LSU Get SN-1 (SN if D-S<0) in lower vector
- subf QW,DR,DBK // IU1 Bytes of full vectors to move (-16)
- cmpi cr7,0,Rt,0xF // IU1 Is Dn right justified?
-
- cmpi cr1,0,D,0 // IU1 Is D0 left justified?
- rlwinm QW,QW,28,4,31 // IU1 Quad words remaining
- add Rt,DST,BC // IU1 Refresh the value of DST+BC
-
- cmpi cr6,0,QW,0 // IU1 Any full vectors to move?
- vperm VPS0,VS0,VS1,VP3 // VPU Align SN-1 and SN to DN
- vor VS1,VS0,VS0 // VIU1 Move lower vector to upper
- beq cr7,Rt_just // b if DN is right justified
-
- mtcrf 0x01,Rt // IU2 Put final vector byte count in cr7
- rlwinm DBK,DBK,0,0,27 // IU1 Address of first byte of final vector
- li D,0 // IU1 Initialize an index pointer
- bnl cr7,Only_1W_bkwd // b if there was only one or zero words to store
-
- stvewx VPS0,DBK,D // LSU store word 1 of two or three
- addi D,D,4 // IU1 increment index
-
- stvewx VPS0,DBK,D // LSU store word 2 of two or three
- addi D,D,4 // IU1 increment index
-Only_1W_bkwd:
- bng cr7,Only_2W_bkwd // b if there were only two or zero words to store
-
- stvewx VPS0,DBK,D // LSU store word 3 of three if necessary
- addi D,D,4 // IU1 increment index
-Only_2W_bkwd:
- bne cr7,Only_B_bkwd // b if there are no half words to store
-
- stvehx VPS0,DBK,D // LSU store one halfword if necessary
- addi D,D,2 // IU1 increment index
-Only_B_bkwd:
- bns cr7,All_done_bkwd // b if there are no bytes to store
-
- stvebx VPS0,DBK,D // LSU store one byte if necessary
- b All_done_bkwd
-
-Rt_just:
- stvx VPS0,DST,BK // LSU Store 16 bytes at DN
-All_done_bkwd:
- addi BK,BK,-16 // IU1 Decrement destination byte count
-
- ble cr6,Last_load // b if no Quad words to do
- mtctr QW // IU2 for (i=0;i<=QW;i++)-execution serializng
- cmpi cr6,0,QW,4 // IU1 Check QW>4
-QW_loop:
- lvx VS0,SRC,BK // LSU Get SN-2 (or SN-1 if ADJ==0)
-
- vperm VPS0,VS0,VS1,VP3 // VPU Align SN-2 and SN-1 to DN-1
- vor VS1,VS0,VS0 // VIU1 Move lower vector to upper
-
- stvx VPS0,DST,BK // LSU Store 16 bytes at DN-1
- addi BK,BK,-16 // IU1 Decrement byte kount
- bdnzf 25,QW_loop // b if 4 or less quad words to do
-
- add DNX,DST,BK // IU1 address of next store (DST+BC-1-16)
- bgt cr6,GT_4QW // b if >4 quad words left
-
-Last_load: // if D-S>=0, next load will be from same address as last
- blt No_ld_bkwd // b if shifting right (S-D>=0)
- addi SRC,SRC,16 // IU1 recorrect source if it was decremented
-No_ld_bkwd:
- lvx VS0,0,SRC // LSU Get last source SN-6 (guaranteed S0)
-// Current 16 bytes is the last; we're done.
- dss 0 // Data stream stop
- vperm VPS0,VS0,VS1,VP3 // VPU Align SN-6 and SN-5 to DN-6
- subfic D,DST,16 // IU1 How many bytes in first destination?
- beq cr1,Lt_just // b if last destination is left justified
-
- mtcrf 0x01,D // IU2 Put byte count remaining in cr7
- li D,0 // IU1 Initialize index pointer
- bns cr7,No_B_bkwd // b if only even number of bytes to store
-
- stvebx VPS0,DST,D // LSU store first byte at DST+0
- addi D,D,1 // IU1 increment index
-No_B_bkwd:
- bne cr7,No_H_bkwd // b if only words to store
- stvehx VPS0,DST,D // LSU store halfword at DST+0/1
- addi D,D,2 // IU1 increment index
-
-No_H_bkwd:
- bng cr7,No_W1_bkwd // b if exactly zero or two words to store
- stvewx VPS0,DST,D // LSU store word 1 of one or three
- addi D,D,4 // IU1 increment index
-
-No_W1_bkwd:
- bnl cr7,No_W2_bkwd // b if there was only one word to store
- stvewx VPS0,DST,D // LSU store word 1 of two or 2 of three
- addi D,D,4 // IU1 increment index
-
- stvewx VPS0,DST,D // LSU store word 2 of two or 3 of three
- b No_W2_bkwd
-
-Lt_just:
- stvx VPS0,0,DST // LSU Store 16 bytes at final dst addr D0
-No_W2_bkwd:
-#ifdef VRSAVE
- mtspr VRSV,RSV // IU1 Restore VRSAVE
-#endif
- blr // Return destination address from entry
-
-GT_4QW: // Do once if next store is to even half of cache line, else twice
-
- lvx VS0,SRC,BK // LSU Get SN-3 (or SN-2)
- mtcrf 0x02,DNX // IU2 cr6[3]=((DST+BC-1)[27]==1)?1:0;
-
- vperm VPS0,VS0,VS1,VP3 // VPU Align SN-3 and SN-2 to Dn-2
- vor VS1,VS0,VS0 // VIU1 Move lower vector to upper
- addi DNX,DNX,-16 // IU1 Prepare to update cr6 next loop
-
- stvx VPS0,DST,BK // LSU Store 16 bytes at DN-2
- vor VS3,VS0,VS0 // VIU Make a copy of lower vector
- addi BK,BK,-16 // IU1 Decrement byte count by 16
- bdnzt 27,GT_4QW // b if next store is to upper (odd) half of CL
-// At this point next store will be to even address.
-
- lis STR,0x102 // IU1 Stream 2 blocks of 16 bytes
- mtcrf 0x02,DST // IU2 cr6[3]=(DST[27]==1)?1:0; (DST odd?)
- addi BL,BK,-16 // IU1 Create an alternate byte count - 16
-
- ori STR,STR,0xffe0 // IU1 Stream stride -32B
- addi SP8,SRC,-64 // IU1 Starting address for data stream touch
- bso cr6,B32_bkwd // b if DST[27] == 1; i.e, final store is odd
-
- bdnz B32_bkwd // decrement counter for last odd QW store
-B32_bkwd: // Should be at least 2 stores remaining and next 2 are cache aligned
- lvx VS2,SRC,BK // LSU Get SN-4 (or SN-3)
- addi SP8,SP8,-32 // IU1 Next starting address for data stream touch
-
- lvx VS1,SRC,BL // LSU Get SN-5 (or SN-4)
- vperm VPS0,VS2,VS3,VP3 // VPU Align SN-4 and SN-3 to DN-3
-
- STRM_1 // LSU Stream 64 byte blocks ahead of loads
-
- DCBL // LSU allocate next cache line
-
- vperm VPS1,VS1,VS2,VP3 // VPU Align SN-5 and SN-4 to DN-4
- vor VS3,VS1,VS1 // VIU1 Move SN-5 to SN-3
-
- stvx VPS0,DST,BK // LSU Store 16 bytes at DN-3
- addi BK,BL,-16 // IU1 Decrement byte count
- bdz Nxt_loc_bkwd // always decrement and branch to next instr
-
-Nxt_loc_bkwd:
- stvx VPS1,DST,BL // LSU Store 16 bytes at DN-4
- addi BL,BK,-16 // IU1 Decrement alternate byte count
- bdnz B32_bkwd // b if there are at least two more QWs to do
-
- bns cr6,One_odd_QW // b if there was one more odd QW to store
- b Last_load
-
-// Come here with two more loads and two stores to do
-One_odd_QW:
- lvx VS1,SRC,BK // LSU Get SN-6 (or SN-5)
-
- vperm VPS1,VS1,VS3,VP3 // VPU Align SN-6 and SN-5 to DN-5
-
- stvx VPS1,DST,BK // LSU Store 16 bytes at DN-5
-
- b Last_load
-
-// End of memmove in AltiVec
-
-#ifdef __MWERKS__
- .align 16
-#else
- .align 4
-#endif
-v_memcpy:
-// Byte count < MIN_VEC bytes will have been copied by scalar code above,
-// so this will not deal with small block moves < MIN_VEC.
-
-#ifdef VRSAVE
- mfspr RSV,VRSV // IU2 Get current VRSAVE contents
-#endif
- rlwinm S,SRC,0,28,31 // IU1 Save src address bits s[28:31]
- rlwinm D,DST,0,28,31 // IU1 D = dst[28:31]
-
-MC_entry: // enter here from memmove if DST-SRC>=BC; this should be faster
-#ifdef VRSAVE
- oris Rt,RSV,0xfff0 // IU1 Or in registers used by this routine
-#endif
- lis BLK,0x010c // IU1 Stream 12 blocks of 16 bytes
-
- subf. S,S,D // IU1 if D-S<0 essentially shifting left
-
-#ifdef VRSAVE
- mtspr VRSV,Rt // IU2 Save in VRSAVE before first vec op
-#endif
- lvsr VP3,0,DMS // LSU Permute vector for dst - src shft right
- ori BLK,BLK,32 // IU1 Stream stride 32B
-
- STRM_F // LSU Start data stream 0 at SRC
- addi DR,DST,16 // IU1 Address of second dst vector
- addi DBK,DBC,-1 // IU1 Address of last dst byte
-
-// If D-S<0 we are "kinda" shifting left with the right shift permute vector
-// loaded to VP3 and we need both S0 and S1 to permute. If D-S>=0 then the
-// first loaded vector needs to be in the upper half of the permute pair and
-// the lower half is a don't care then.
- bge Ld_bytes_rt // b if shifting right (D-S>=0)
-
- lvx VS0,0,SRC // LSU Get S0 load started
-// Comments numbering source and destination assume single path through the
-// code executing each instruction once. For vec_memcpy, an example would
-// be the call memcpy(BASE+0x1E, BASE+0x1F, 259). N = 16 in that case.
- addi SRC,SRC,16 // IU1 Increment src base (to keep BK useful)
-
-Ld_bytes_rt: // Come here to get VS1 & Don't care what VS0 is
- lvx VS1,0,SRC // LSU Get S1 (or S0 if D-S>=0) in upper vector
- rlwinm DR,DR,0,0,27 // IU1 (DST+16)[0:27]
- cmpi cr1,0,D,0 // IU1 Is D0 left justified?
-
- subf Rt,DST,DR // IU1 How many bytes in first destination?
- subf QW,DR,DBK // IU1 Bytes of full vectors to move (-16)
- li BK,0 // IU1 Initialize byte kount index
-
- mtcrf 0x01,Rt // IU2 Put bytes in 1st dst in cr7
- rlwinm QW,QW,28,4,31 // IU1 Quad words remaining
- vperm VPS0,VS0,VS1,VP3 // VPU Align S0 and S1 to D0
-
- vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
- beq cr1,Left_just // b if D0 is left justified
-
- bns cr7,No_B_fwd // b if only even number of bytes to store
-
- stvebx VPS0,DST,BK // LSU store first byte at DST+0
- addi BK,BK,1 // IU1 increment index
-No_B_fwd:
- bne cr7,No_H_fwd // b if only words to store
-
- stvehx VPS0,DST,BK // LSU store halfword at DST+0/1
- addi BK,BK,2 // IU1 increment index
-No_H_fwd:
- bng cr7,No_W1_fwd // b if exactly zero or two words to store
-
- stvewx VPS0,DST,BK // LSU store word 1 of one or three
- addi BK,BK,4 // IU1 increment index
-
-No_W1_fwd:
- bnl cr7,No_W2_fwd // b if there was only one word to store
- stvewx VPS0,DST,BK // LSU store word 1 of two or 2 of three
- addi BK,BK,4 // IU1 increment index
-
- stvewx VPS0,DST,BK // LSU store word 2 of two or 3 of three
- b No_W2_fwd
-
-Left_just:
- stvx VPS0,0,DST // LSU Store 16 bytes at D0
-No_W2_fwd:
- rlwinm Rt,DBK,0,28,31 // IU1 (DBK = DST+BC-1)[28:31]
- cmpi cr6,0,QW,0 // IU1 Any full vectors to move?
-
- li BK,16 // IU1 Re-initialize byte kount index
- cmpi cr1,0,Rt,0xF // IU1 Is DN right justified?
- cmpi cr7,0,QW,14 // IU1 Check QW>14
- ble cr6,Last_ld_fwd // b if no Quad words to do
-
- mtctr QW // IU2 for (i=0;i<=QW;i++)
- cmpi cr6,0,QW,4 // IU1 Check QW>4
-QW_fwd_loop:
- lvx VS1,SRC,BK // LSU Get S2 (or S1)
-
- vperm VPS0,VS0,VS1,VP3 // VPU Align S1 and S2 to D1
- vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
-
- stvx VPS0,DST,BK // LSU Store 16 bytes at D1(+n*16 where n<4)
- addi BK,BK,16 // IU1 Increment byte kount index
- bdnzf 25,QW_fwd_loop // b if 4 or less quad words to do
-
- add DNX,DST,BK // IU1 address of next store (DST+32 if QW>4)
- addi QW,QW,-1 // IU1 One more QW stored by now
- bgt cr6,GT_4QW_fwd // b if >4 quad words left
-
-Last_ld_fwd: // Next 16 bytes is the last; we're done.
- add DBC,DST,BC // IU1 Recompute address of last dst byte + 1
- add SBC,SRC,BC // IU1 Recompute address of last src byte + 1
- bge No_ld_fwd // b if shifting right (D-S>=0)
-
- addi SBC,SBC,-16 // IU1 if D-S>=0 we didn't add 16 to src
-No_ld_fwd:
- mtcrf 0x01,DBC // IU2 Put final vector byte count in cr7
- addi DBK,DBC,-1 // IU1 Recompute address of last dst byte
- addi Rt,SBC,-1 // IU1 Recompute address of last src byte
-
-// If D-S<0 we have already loaded all the source vectors.
-// If D-S>=0 then the first loaded vector went to the upper half of the permute
-// pair and we need one more vector. (This may be a duplicate.)
-
- lvx VS1,0,Rt // LSU Get last source S14 (guaranteed SN)
-
-#ifndef NO_DST
- dss 0 // Data stream 0 stop
-
- dss 1 // Data stream 1 stop
-#endif
- vperm VPS0,VS0,VS1,VP3 // VPU Align S13 and S14 to D14
- beq cr1,Rt_just_fwd // b if last destination is right justified
-
- rlwinm DBK,DBK,0,0,27 // IU1 Round to QW addr of last byte
- li D,0 // IU1 Initialize index pointer
- bnl cr7,Only_1W_fwd // b if there was only one or zero words to store
-
- stvewx VPS0,DBK,D // LSU store word 1 of two or three
- addi D,D,4 // IU1 increment index
-
- stvewx VPS0,DBK,D // LSU store word 2 of two or three
- addi D,D,4 // IU1 increment index
-Only_1W_fwd:
- bng cr7,Only_2W_fwd // b if there were only two or zero words to store
-
- stvewx VPS0,DBK,D // LSU store word 3 of three if necessary
- addi D,D,4 // IU1 increment index
-Only_2W_fwd:
- bne cr7,Only_B_fwd // b if there are no half words to store
-
- stvehx VPS0,DBK,D // LSU store one halfword if necessary
- addi D,D,2 // IU1 increment index
-Only_B_fwd:
- bns cr7,All_done_fwd // b if there are no bytes to store
-
- stvebx VPS0,DBK,D // LSU store one byte if necessary
- b All_done_fwd
-
-Rt_just_fwd:
-
- stvx VPS0,DST,BK // LSU Store 16 bytes at D14
-All_done_fwd:
-#ifdef VRSAVE
- mtspr VRSV,RSV // IU1 Restore VRSAVE
-#endif
- blr // Return destination address from entry
-#ifdef __MWERKS__
- .align 16
-#else
- .align 4
-#endif
-GT_4QW_fwd: // Do once if nxt st is to odd half of cache line, else twice
-
- lvx VS1,SRC,BK // LSU Get S3 (or S2)
- addi QW,QW,-1 // IU1 Keeping track of QWs stored
- mtcrf 0x02,DNX // IU2 cr6[3]=((DST+32)[27]==1)?1:0;
-
- addi DNX,DNX,16 // IU1 Update cr6 for next loop
- addi Rt,QW,-2 // IU1 Insure at least 2 QW left after big loop
-
- vperm VPS0,VS0,VS1,VP3 // VPU Align S2 and S3 to D2
- vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
-
- stvx VPS0,DST,BK // LSU Store 16 bytes at D2
- addi BK,BK,16 // IU1 Increment byte count by 16
- bdnzf 27,GT_4QW_fwd // b if next store is to lower (even) half of CL
-// At this point next store will be to even address.
-
- mtcrf 0x02,DBK // IU2 cr6[3]=((last store)[27]==1)?1:0; (odd?)
- lis STR,0x104 // IU1 Stream 4 blocks of 16 bytes
- addi BL,BK,16 // IU1 Create an alternate byte kount + 32
-
- ori STR,STR,32 // IU1 Stream stride 32B
-#ifndef NO_BIG_LOOP
- rlwinm BIG,Rt,29,3,31 // IU1 QW/8 big loops to do
-
- rlwinm Rt,Rt,0,0,28 // IU1 How many QWs will be done in big loop
- bgt cr7,Big_loop // b if QW > 14
-#endif
-No_big_loop:
-// We need the ctr register to reflect an even byte count before entering
-// the next block - faster to decrement than to reload.
-
- addi SP8,SRC,256 // IU1 Starting address for data stream touch
- xoris STR,STR,0x6 // IU1 Reset stream to 2 blocks of 16 bytes
- bns cr6,B32_fwd // b if DST[27] == 0; i.e, final store is even
-
- bdnz B32_fwd // decrement counter for last QW store odd
-
-B32_fwd: // Should be at least 2 stores remaining and next 2 are cache aligned
- lvx VS1,SRC,BK // LSU Get S12
- addi SP8,SP8,32 // IU1 Next starting address for data stream touch
-
- lvx VS2,SRC,BL // LSU Get S13
- vperm VPS1,VS0,VS1,VP3 // VPU Align S11 and S12 to D11
-
- STRM_1 // LSU Stream 64 byte blocks ahead of loads
-
- DCBK // LSU then Kill instead of RWITM
-
- vperm VPS0,VS1,VS2,VP3 // VPU Align S12 and S13 to D12
- vor VS0,VS2,VS2 // VIU1 Move S13 to S11
-
- stvx VPS1,DST,BK // LSU Store 16 bytes at D11
- addi BK,BL,16 // IU1 Increment byte count
- bdz Nxt_loc_fwd // always decrement and branch to next instr
-
-Nxt_loc_fwd:
- stvx VPS0,DST,BL // LSU Store 16 bytes at D12
- addi BL,BK,16 // IU1 Increment alternate byte count
- bdnz B32_fwd // b if there are at least two more QWs to do
-
- bso cr6,One_even_QW // b if there is one even and one odd QW to store
- b Last_ld_fwd // b if last store is to even address
-
-// Come here with two more loads and two stores to do
-One_even_QW:
- lvx VS1,SRC,BK // LSU Get S14 (or S13 if if D-S>=0)
-
- vperm VPS0,VS0,VS1,VP3 // VPU Align S13 and S14 to D13
- vor VS0,VS1,VS1 // VIU1 Move upper vector to lower
-
- stvx VPS0,DST,BK // LSU Store 16 bytes at D13
- addi BK,BK,16 // IU1 Increment byte count
-
- b Last_ld_fwd
-
-#ifdef __MWERKS__
- .align 16
-#else
- .align 4
-#endif
-Big_loop:
- subf QW,Rt,QW // IU1 Should be 2-7 QWs left after big loop
- blt cr5,No_big_loop // b back if |DST-SRC|<128; Big_loop won't work.
- mtctr BIG // IU2 loop for as many 128B loops as possible
- addi SP8,SRC,256 // IU1 Starting address for data stream touch
-
-Loop_of_128B: // Come here with QW>=10 and next store even; VS0 last load
- lvx VS1,SRC,BK // LSU Get S4 (or S3 if D-S>=0)
- addi BL,BK,32 // IU1 Increment Byte_Kount+16 by 32
- addi SP8,SP8,128 // IU1 increment address for data stream touch
-
- lvx VS3,SRC,BL // LSU Get S6 (or S5)
- addi BL,BL,32 // IU1 Increment Byte_Kount+48 by 32
-
- lvx VS5,SRC,BL // LSU Get S8 (or S7)
- addi BL,BL,32 // IU1 Increment Byte_Kount+80 by 32
-
- lvx VS7,SRC,BL // LSU Get S10 (or S9)
- addi BL,BK,16 // IU1 Increment Byte_Kount+16 by 16
-
- lvx VS2,SRC,BL // LSU Get S5 (or S4)
- addi BL,BL,32 // IU1 Increment Byte_Kount+32 by 32
-
- lvx VS4,SRC,BL // LSU Get S7 (or S6)
- addi BL,BL,32 // IU1 Increment Byte_Kount+64 by 32
-
- lvx VS6,SRC,BL // LSU Get S9 (or S8)
- addi BL,BL,32 // IU1 Increment Byte_Kount+96 by 32
- vperm VPS0,VS0,VS1,VP3 // VPU
-
- lvx VS0,SRC,BL // LSU Get S11 (or S10)
- vperm VPS1,VS1,VS2,VP3 // VPU
-
- STRM_1 // LSU Stream 4 32B blocks, stride 32B
-
- DCBK // LSU then Kill instead of RWITM
-
- stvx VPS0,DST,BK // LSU Store D3
- addi BK,BK,16 // IU1 Increment Byte_Kount+16 by 16
- vperm VPS2,VS2,VS3,VP3 // VPU
-
- stvx VPS1,DST,BK // LSU Store D4
- addi BK,BK,16 // IU1 Increment Byte_Kount+32 by 16
- vperm VPS3,VS3,VS4,VP3 // VPU
-
- DCBK // LSU then Kill instead of RWITM
-
- stvx VPS2,DST,BK // LSU Store D5
- addi BK,BK,16 // IU1 Increment Byte_Kount+48 by 16
- vperm VPS4,VS4,VS5,VP3 // VPU
-
- stvx VPS3,DST,BK // LSU Store D6
- addi BK,BK,16 // IU1 Increment Byte_Kount+64 by 16
- vperm VPS5,VS5,VS6,VP3 // VPU
-
- DCBK // LSU then Kill instead of RWITM
-
- stvx VPS4,DST,BK // LSU Store D7
- addi BK,BK,16 // IU1 Increment Byte_Kount+80 by 16
- vperm VPS6,VS6,VS7,VP3 // VPU
-
- stvx VPS5,DST,BK // LSU Store D8
- addi BK,BK,16 // IU1 Increment Byte_Kount+96 by 16
- vperm VPS7,VS7,VS0,VP3 // VPU
-
- DCBK // LSU then Kill instead of RWITM
-
- stvx VPS6,DST,BK // LSU Store D9
- addi BK,BK,16 // IU1 Increment Byte_Kount+112 by 16
-
- stvx VPS7,DST,BK // LSU Store D10
- addi BK,BK,16 // IU1 Increment Byte_Kount+128 by 16
- bdnz Loop_of_128B // b if ctr > 0 (QW/8 still > 0)
-
- mtctr QW // IU1 Restore QW remaining to counter
- addi BL,BK,16 // IU1 Create an alternate byte kount + 16
- bns cr6,B32_fwd // b if DST[27] == 0; i.e, final store is even
-
- bdnz B32_fwd // b and decrement counter for last QW store odd
- // One of the above branches should have taken
-
-// End of memcpy in AltiVec
-
-// bcopy works like memcpy, but the source and destination operands are reversed.
-// Following will just reverse the operands and branch to memcpy.
-
-#ifdef LIBMOTOVEC
- .globl bcopy
-bcopy:
-#else
- .globl vec_bcopy
-vec_bcopy:
-#endif
- mr Rt,DST // temp storage for what is really source address (r3)
- mr DST,SRC // swap destination address to r3 to match memcpy dst
- mr SRC,Rt // Complete swap of destination and source for memcpy
-#ifdef LIBMOTOVEC
- b memcpy // b to memcpy with correct args in r3 and r4
-#else
- b _vec_memcpy // b to vec_memcpy with correct args in r3 and r4
-#endif
-// End of bcopy in AltiVec