1 #!/usr/bin/env python

     2 # -*- coding: iso-8859-1 -*-

     3 #-------------------------------------------------------------------

     4 # tarfile.py

     5 #-------------------------------------------------------------------

     6 # Copyright (C) 2002 Lars Gustäbel <lars@gustaebel.de>

     7 # All rights reserved.

     8 #

     9 # Permission  is  hereby granted,  free  of charge,  to  any person

    10 # obtaining a  copy of  this software  and associated documentation

    11 # files  (the  "Software"),  to   deal  in  the  Software   without

    12 # restriction,  including  without limitation  the  rights to  use,

    13 # copy, modify, merge, publish, distribute, sublicense, and/or sell

    14 # copies  of  the  Software,  and to  permit  persons  to  whom the

    15 # Software  is  furnished  to  do  so,  subject  to  the  following

    16 # conditions:

    17 #

    18 # The above copyright  notice and this  permission notice shall  be

    19 # included in all copies or substantial portions of the Software.

    20 #

    21 # THE SOFTWARE IS PROVIDED "AS  IS", WITHOUT WARRANTY OF ANY  KIND,

    22 # EXPRESS OR IMPLIED, INCLUDING  BUT NOT LIMITED TO  THE WARRANTIES

    23 # OF  MERCHANTABILITY,  FITNESS   FOR  A  PARTICULAR   PURPOSE  AND

    24 # NONINFRINGEMENT.  IN  NO  EVENT SHALL  THE  AUTHORS  OR COPYRIGHT

    25 # HOLDERS  BE LIABLE  FOR ANY  CLAIM, DAMAGES  OR OTHER  LIABILITY,

    26 # WHETHER  IN AN  ACTION OF  CONTRACT, TORT  OR OTHERWISE,  ARISING

    27 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR

    28 # OTHER DEALINGS IN THE SOFTWARE.

    29 #

    30 """Read from and write to tar format archives.

    31 """

    32 

    33 __version__ = "$Revision: 60730 $"

    34 # $Source$

    35 

    36 version     = "0.8.0"

    37 __author__  = "Lars Gustäbel (lars@gustaebel.de)"

    38 __date__    = "$Date: 2008-02-11 19:36:07 +0100 (Mo, 11 Feb 2008) $"

    39 __cvsid__   = "$Id: tarfile.py 60730 2008-02-11 18:36:07Z lars.gustaebel $"

    40 __credits__ = "Gustavo Niemeyer, Niels Gustäbel, Richard Townsend."

    41 

    42 #---------

    43 # Imports

    44 #---------

    45 import sys

    46 import os

    47 import shutil

    48 import stat

    49 import errno

    50 import time

    51 import struct

    52 import copy

    53 

    54 if sys.platform == 'mac':

    55     # This module needs work for MacOS9, especially in the area of pathname

    56     # handling. In many places it is assumed a simple substitution of / by the

    57     # local os.path.sep is good enough to convert pathnames, but this does not

    58     # work with the mac rooted:path:name versus :nonrooted:path:name syntax

    59     raise ImportError, "tarfile does not work for platform==mac"

    60 

    61 try:

    62     import grp, pwd

    63 except ImportError:

    64     grp = pwd = None

    65 

    66 # from tarfile import *

    67 __all__ = ["TarFile", "TarInfo", "is_tarfile", "TarError"]

    68 

    69 #---------------------------------------------------------

    70 # tar constants

    71 #---------------------------------------------------------

    72 NUL        = "\0"               # the null character

    73 BLOCKSIZE  = 512                # length of processing blocks

    74 RECORDSIZE = BLOCKSIZE * 20     # length of records

    75 MAGIC      = "ustar"            # magic tar string

    76 VERSION    = "00"               # version number

    77 

    78 LENGTH_NAME    = 100            # maximum length of a filename

    79 LENGTH_LINK    = 100            # maximum length of a linkname

    80 LENGTH_PREFIX  = 155            # maximum length of the prefix field

    81 MAXSIZE_MEMBER = 077777777777L  # maximum size of a file (11 octal digits)

    82 

    83 REGTYPE  = "0"                  # regular file

    84 AREGTYPE = "\0"                 # regular file

    85 LNKTYPE  = "1"                  # link (inside tarfile)

    86 SYMTYPE  = "2"                  # symbolic link

    87 CHRTYPE  = "3"                  # character special device

    88 BLKTYPE  = "4"                  # block special device

    89 DIRTYPE  = "5"                  # directory

    90 FIFOTYPE = "6"                  # fifo special device

    91 CONTTYPE = "7"                  # contiguous file

    92 

    93 GNUTYPE_LONGNAME = "L"          # GNU tar extension for longnames

    94 GNUTYPE_LONGLINK = "K"          # GNU tar extension for longlink

    95 GNUTYPE_SPARSE   = "S"          # GNU tar extension for sparse file

    96 

    97 #---------------------------------------------------------

    98 # tarfile constants

    99 #---------------------------------------------------------

   100 SUPPORTED_TYPES = (REGTYPE, AREGTYPE, LNKTYPE,  # file types that tarfile

   101                    SYMTYPE, DIRTYPE, FIFOTYPE,  # can cope with.

   102                    CONTTYPE, CHRTYPE, BLKTYPE,

   103                    GNUTYPE_LONGNAME, GNUTYPE_LONGLINK,

   104                    GNUTYPE_SPARSE)

   105 

   106 REGULAR_TYPES = (REGTYPE, AREGTYPE,             # file types that somehow

   107                  CONTTYPE, GNUTYPE_SPARSE)      # represent regular files

   108 

   109 #---------------------------------------------------------

   110 # Bits used in the mode field, values in octal.

   111 #---------------------------------------------------------

   112 S_IFLNK = 0120000        # symbolic link

   113 S_IFREG = 0100000        # regular file

   114 S_IFBLK = 0060000        # block device

   115 S_IFDIR = 0040000        # directory

   116 S_IFCHR = 0020000        # character device

   117 S_IFIFO = 0010000        # fifo

   118 

   119 TSUID   = 04000          # set UID on execution

   120 TSGID   = 02000          # set GID on execution

   121 TSVTX   = 01000          # reserved

   122 

   123 TUREAD  = 0400           # read by owner

   124 TUWRITE = 0200           # write by owner

   125 TUEXEC  = 0100           # execute/search by owner

   126 TGREAD  = 0040           # read by group

   127 TGWRITE = 0020           # write by group

   128 TGEXEC  = 0010           # execute/search by group

   129 TOREAD  = 0004           # read by other

   130 TOWRITE = 0002           # write by other

   131 TOEXEC  = 0001           # execute/search by other

   132 

   133 #---------------------------------------------------------

   134 # Some useful functions

   135 #---------------------------------------------------------

   136 

   137 def stn(s, length):

   138     """Convert a python string to a null-terminated string buffer.

   139     """

   140     return s[:length] + (length - len(s)) * NUL

   141 

   142 def nts(s):

   143     """Convert a null-terminated string field to a python string.

   144     """

   145     # Use the string up to the first null char.

   146     p = s.find("\0")

   147     if p == -1:

   148         return s

   149     return s[:p]

   150 

   151 def nti(s):

   152     """Convert a number field to a python number.

   153     """

   154     # There are two possible encodings for a number field, see

   155     # itn() below.

   156     if s[0] != chr(0200):

   157         n = int(nts(s) or "0", 8)

   158     else:

   159         n = 0L

   160         for i in xrange(len(s) - 1):

   161             n <<= 8

   162             n += ord(s[i + 1])

   163     return n

   164 

   165 def itn(n, digits=8, posix=False):

   166     """Convert a python number to a number field.

   167     """

   168     # POSIX 1003.1-1988 requires numbers to be encoded as a string of

   169     # octal digits followed by a null-byte, this allows values up to

   170     # (8**(digits-1))-1. GNU tar allows storing numbers greater than

   171     # that if necessary. A leading 0200 byte indicates this particular

   172     # encoding, the following digits-1 bytes are a big-endian

   173     # representation. This allows values up to (256**(digits-1))-1.

   174     if 0 <= n < 8 ** (digits - 1):

   175         s = "%0*o" % (digits - 1, n) + NUL

   176     else:

   177         if posix:

   178             raise ValueError("overflow in number field")

   179 

   180         if n < 0:

   181             # XXX We mimic GNU tar's behaviour with negative numbers,

   182             # this could raise OverflowError.

   183             n = struct.unpack("L", struct.pack("l", n))[0]

   184 

   185         s = ""

   186         for i in xrange(digits - 1):

   187             s = chr(n & 0377) + s

   188             n >>= 8

   189         s = chr(0200) + s

   190     return s

   191 

   192 def calc_chksums(buf):

   193     """Calculate the checksum for a member's header by summing up all

   194        characters except for the chksum field which is treated as if

   195        it was filled with spaces. According to the GNU tar sources,

   196        some tars (Sun and NeXT) calculate chksum with signed char,

   197        which will be different if there are chars in the buffer with

   198        the high bit set. So we calculate two checksums, unsigned and

   199        signed.

   200     """

   201     unsigned_chksum = 256 + sum(struct.unpack("148B", buf[:148]) + struct.unpack("356B", buf[156:512]))

   202     signed_chksum = 256 + sum(struct.unpack("148b", buf[:148]) + struct.unpack("356b", buf[156:512]))

   203     return unsigned_chksum, signed_chksum

   204 

   205 def copyfileobj(src, dst, length=None):

   206     """Copy length bytes from fileobj src to fileobj dst.

   207        If length is None, copy the entire content.

   208     """

   209     if length == 0:

   210         return

   211     if length is None:

   212         shutil.copyfileobj(src, dst)

   213         return

   214 

   215     BUFSIZE = 16 * 1024

   216     blocks, remainder = divmod(length, BUFSIZE)

   217     for b in xrange(blocks):

   218         buf = src.read(BUFSIZE)

   219         if len(buf) < BUFSIZE:

   220             raise IOError("end of file reached")

   221         dst.write(buf)

   222 

   223     if remainder != 0:

   224         buf = src.read(remainder)

   225         if len(buf) < remainder:

   226             raise IOError("end of file reached")

   227         dst.write(buf)

   228     return

   229 

   230 filemode_table = (

   231     ((S_IFLNK,      "l"),

   232      (S_IFREG,      "-"),

   233      (S_IFBLK,      "b"),

   234      (S_IFDIR,      "d"),

   235      (S_IFCHR,      "c"),

   236      (S_IFIFO,      "p")),

   237 

   238     ((TUREAD,       "r"),),

   239     ((TUWRITE,      "w"),),

   240     ((TUEXEC|TSUID, "s"),

   241      (TSUID,        "S"),

   242      (TUEXEC,       "x")),

   243 

   244     ((TGREAD,       "r"),),

   245     ((TGWRITE,      "w"),),

   246     ((TGEXEC|TSGID, "s"),

   247      (TSGID,        "S"),

   248      (TGEXEC,       "x")),

   249 

   250     ((TOREAD,       "r"),),

   251     ((TOWRITE,      "w"),),

   252     ((TOEXEC|TSVTX, "t"),

   253      (TSVTX,        "T"),

   254      (TOEXEC,       "x"))

   255 )

   256 

   257 def filemode(mode):

   258     """Convert a file's mode to a string of the form

   259        -rwxrwxrwx.

   260        Used by TarFile.list()

   261     """

   262     perm = []

   263     for table in filemode_table:

   264         for bit, char in table:

   265             if mode & bit == bit:

   266                 perm.append(char)

   267                 break

   268         else:

   269             perm.append("-")

   270     return "".join(perm)

   271 

   272 if os.sep != "/":

   273     normpath = lambda path: os.path.normpath(path).replace(os.sep, "/")

   274 else:

   275     normpath = os.path.normpath

   276 

   277 class TarError(Exception):

   278     """Base exception."""

   279     pass

   280 class ExtractError(TarError):

   281     """General exception for extract errors."""

   282     pass

   283 class ReadError(TarError):

   284     """Exception for unreadble tar archives."""

   285     pass

   286 class CompressionError(TarError):

   287     """Exception for unavailable compression methods."""

   288     pass

   289 class StreamError(TarError):

   290     """Exception for unsupported operations on stream-like TarFiles."""

   291     pass

   292 

   293 #---------------------------

   294 # internal stream interface

   295 #---------------------------

   296 class _LowLevelFile:

   297     """Low-level file object. Supports reading and writing.

   298        It is used instead of a regular file object for streaming

   299        access.

   300     """

   301 

   302     def __init__(self, name, mode):

   303         mode = {

   304             "r": os.O_RDONLY,

   305             "w": os.O_WRONLY | os.O_CREAT | os.O_TRUNC,

   306         }[mode]

   307         if hasattr(os, "O_BINARY"):

   308             mode |= os.O_BINARY

   309         self.fd = os.open(name, mode)

   310 

   311     def close(self):

   312         os.close(self.fd)

   313 

   314     def read(self, size):

   315         return os.read(self.fd, size)

   316 

   317     def write(self, s):

   318         os.write(self.fd, s)

   319 

   320 class _Stream:

   321     """Class that serves as an adapter between TarFile and

   322        a stream-like object.  The stream-like object only

   323        needs to have a read() or write() method and is accessed

   324        blockwise.  Use of gzip or bzip2 compression is possible.

   325        A stream-like object could be for example: sys.stdin,

   326        sys.stdout, a socket, a tape device etc.

   327 

   328        _Stream is intended to be used only internally.

   329     """

   330 

   331     def __init__(self, name, mode, comptype, fileobj, bufsize):

   332         """Construct a _Stream object.

   333         """

   334         self._extfileobj = True

   335         if fileobj is None:

   336             fileobj = _LowLevelFile(name, mode)

   337             self._extfileobj = False

   338 

   339         if comptype == '*':

   340             # Enable transparent compression detection for the

   341             # stream interface

   342             fileobj = _StreamProxy(fileobj)

   343             comptype = fileobj.getcomptype()

   344 

   345         self.name     = name or ""

   346         self.mode     = mode

   347         self.comptype = comptype

   348         self.fileobj  = fileobj

   349         self.bufsize  = bufsize

   350         self.buf      = ""

   351         self.pos      = 0L

   352         self.closed   = False

   353 

   354         if comptype == "gz":

   355             try:

   356                 import zlib

   357             except ImportError:

   358                 raise CompressionError("zlib module is not available")

   359             self.zlib = zlib

   360             self.crc = zlib.crc32("")

   361             if mode == "r":

   362                 self._init_read_gz()

   363             else:

   364                 self._init_write_gz()

   365 

   366         if comptype == "bz2":

   367             try:

   368                 import bz2

   369             except ImportError:

   370                 raise CompressionError("bz2 module is not available")

   371             if mode == "r":

   372                 self.dbuf = ""

   373                 self.cmp = bz2.BZ2Decompressor()

   374             else:

   375                 self.cmp = bz2.BZ2Compressor()

   376 

   377     def __del__(self):

   378         if hasattr(self, "closed") and not self.closed:

   379             self.close()

   380 

   381     def _init_write_gz(self):

   382         """Initialize for writing with gzip compression.

   383         """

   384         self.cmp = self.zlib.compressobj(9, self.zlib.DEFLATED,

   385                                             -self.zlib.MAX_WBITS,

   386                                             self.zlib.DEF_MEM_LEVEL,

   387                                             0)

   388         timestamp = struct.pack("<L", long(time.time()))

   389         self.__write("\037\213\010\010%s\002\377" % timestamp)

   390         if self.name.endswith(".gz"):

   391             self.name = self.name[:-3]

   392         self.__write(self.name + NUL)

   393 

   394     def write(self, s):

   395         """Write string s to the stream.

   396         """

   397         if self.comptype == "gz":

   398             self.crc = self.zlib.crc32(s, self.crc)

   399         self.pos += len(s)

   400         if self.comptype != "tar":

   401             s = self.cmp.compress(s)

   402         self.__write(s)

   403 

   404     def __write(self, s):

   405         """Write string s to the stream if a whole new block

   406            is ready to be written.

   407         """

   408         self.buf += s

   409         while len(self.buf) > self.bufsize:

   410             self.fileobj.write(self.buf[:self.bufsize])

   411             self.buf = self.buf[self.bufsize:]

   412 

   413     def close(self):

   414         """Close the _Stream object. No operation should be

   415            done on it afterwards.

   416         """

   417         if self.closed:

   418             return

   419 

   420         if self.mode == "w" and self.comptype != "tar":

   421             self.buf += self.cmp.flush()

   422 

   423         if self.mode == "w" and self.buf:

   424             self.fileobj.write(self.buf)

   425             self.buf = ""

   426             if self.comptype == "gz":

   427                 # The native zlib crc is an unsigned 32-bit integer, but

   428                 # the Python wrapper implicitly casts that to a signed C

   429                 # long.  So, on a 32-bit box self.crc may "look negative",

   430                 # while the same crc on a 64-bit box may "look positive".

   431                 # To avoid irksome warnings from the `struct` module, force

   432                 # it to look positive on all boxes.

   433                 self.fileobj.write(struct.pack("<L", self.crc & 0xffffffffL))

   434                 self.fileobj.write(struct.pack("<L", self.pos & 0xffffFFFFL))

   435 

   436         if not self._extfileobj:

   437             self.fileobj.close()

   438 

   439         self.closed = True

   440 

   441     def _init_read_gz(self):

   442         """Initialize for reading a gzip compressed fileobj.

   443         """

   444         self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS)

   445         self.dbuf = ""

   446 

   447         # taken from gzip.GzipFile with some alterations

   448         if self.__read(2) != "\037\213":

   449             raise ReadError("not a gzip file")

   450         if self.__read(1) != "\010":

   451             raise CompressionError("unsupported compression method")

   452 

   453         flag = ord(self.__read(1))

   454         self.__read(6)

   455 

   456         if flag & 4:

   457             xlen = ord(self.__read(1)) + 256 * ord(self.__read(1))

   458             self.read(xlen)

   459         if flag & 8:

   460             while True:

   461                 s = self.__read(1)

   462                 if not s or s == NUL:

   463                     break

   464         if flag & 16:

   465             while True:

   466                 s = self.__read(1)

   467                 if not s or s == NUL:

   468                     break

   469         if flag & 2:

   470             self.__read(2)

   471 

   472     def tell(self):

   473         """Return the stream's file pointer position.

   474         """

   475         return self.pos

   476 

   477     def seek(self, pos=0):

   478         """Set the stream's file pointer to pos. Negative seeking

   479            is forbidden.

   480         """

   481         if pos - self.pos >= 0:

   482             blocks, remainder = divmod(pos - self.pos, self.bufsize)

   483             for i in xrange(blocks):

   484                 self.read(self.bufsize)

   485             self.read(remainder)

   486         else:

   487             raise StreamError("seeking backwards is not allowed")

   488         return self.pos

   489 

   490     def read(self, size=None):

   491         """Return the next size number of bytes from the stream.

   492            If size is not defined, return all bytes of the stream

   493            up to EOF.

   494         """

   495         if size is None:

   496             t = []

   497             while True:

   498                 buf = self._read(self.bufsize)

   499                 if not buf:

   500                     break

   501                 t.append(buf)

   502             buf = "".join(t)

   503         else:

   504             buf = self._read(size)

   505         self.pos += len(buf)

   506         return buf

   507 

   508     def _read(self, size):

   509         """Return size bytes from the stream.

   510         """

   511         if self.comptype == "tar":

   512             return self.__read(size)

   513 

   514         c = len(self.dbuf)

   515         t = [self.dbuf]

   516         while c < size:

   517             buf = self.__read(self.bufsize)

   518             if not buf:

   519                 break

   520             buf = self.cmp.decompress(buf)

   521             t.append(buf)

   522             c += len(buf)

   523         t = "".join(t)

   524         self.dbuf = t[size:]

   525         return t[:size]

   526 

   527     def __read(self, size):

   528         """Return size bytes from stream. If internal buffer is empty,

   529            read another block from the stream.

   530         """

   531         c = len(self.buf)

   532         t = [self.buf]

   533         while c < size:

   534             buf = self.fileobj.read(self.bufsize)

   535             if not buf:

   536                 break

   537             t.append(buf)

   538             c += len(buf)

   539         t = "".join(t)

   540         self.buf = t[size:]

   541         return t[:size]

   542 # class _Stream

   543 

   544 class _StreamProxy(object):

   545     """Small proxy class that enables transparent compression

   546        detection for the Stream interface (mode 'r|*').

   547     """

   548 

   549     def __init__(self, fileobj):

   550         self.fileobj = fileobj

   551         self.buf = self.fileobj.read(BLOCKSIZE)

   552 

   553     def read(self, size):

   554         self.read = self.fileobj.read

   555         return self.buf

   556 

   557     def getcomptype(self):

   558         if self.buf.startswith("\037\213\010"):

   559             return "gz"

   560         if self.buf.startswith("BZh91"):

   561             return "bz2"

   562         return "tar"

   563 

   564     def close(self):

   565         self.fileobj.close()

   566 # class StreamProxy

   567 

   568 class _BZ2Proxy(object):

   569     """Small proxy class that enables external file object

   570        support for "r:bz2" and "w:bz2" modes. This is actually

   571        a workaround for a limitation in bz2 module's BZ2File

   572        class which (unlike gzip.GzipFile) has no support for

   573        a file object argument.

   574     """

   575 

   576     blocksize = 16 * 1024

   577 

   578     def __init__(self, fileobj, mode):

   579         self.fileobj = fileobj

   580         self.mode = mode

   581         self.init()

   582 

   583     def init(self):

   584         import bz2

   585         self.pos = 0

   586         if self.mode == "r":

   587             self.bz2obj = bz2.BZ2Decompressor()

   588             self.fileobj.seek(0)

   589             self.buf = ""

   590         else:

   591             self.bz2obj = bz2.BZ2Compressor()

   592 

   593     def read(self, size):

   594         b = [self.buf]

   595         x = len(self.buf)

   596         while x < size:

   597             try:

   598                 raw = self.fileobj.read(self.blocksize)

   599                 data = self.bz2obj.decompress(raw)

   600                 b.append(data)

   601             except EOFError:

   602                 break

   603             x += len(data)

   604         self.buf = "".join(b)

   605 

   606         buf = self.buf[:size]

   607         self.buf = self.buf[size:]

   608         self.pos += len(buf)

   609         return buf

   610 

   611     def seek(self, pos):

   612         if pos < self.pos:

   613             self.init()

   614         self.read(pos - self.pos)

   615 

   616     def tell(self):

   617         return self.pos

   618 

   619     def write(self, data):

   620         self.pos += len(data)

   621         raw = self.bz2obj.compress(data)

   622         self.fileobj.write(raw)

   623 

   624     def close(self):

   625         if self.mode == "w":

   626             raw = self.bz2obj.flush()

   627             self.fileobj.write(raw)

   628         self.fileobj.close()

   629 # class _BZ2Proxy

   630 

   631 #------------------------

   632 # Extraction file object

   633 #------------------------

   634 class _FileInFile(object):

   635     """A thin wrapper around an existing file object that

   636        provides a part of its data as an individual file

   637        object.

   638     """

   639 

   640     def __init__(self, fileobj, offset, size, sparse=None):

   641         self.fileobj = fileobj

   642         self.offset = offset

   643         self.size = size

   644         self.sparse = sparse

   645         self.position = 0

   646 

   647     def tell(self):

   648         """Return the current file position.

   649         """

   650         return self.position

   651 

   652     def seek(self, position):

   653         """Seek to a position in the file.

   654         """

   655         self.position = position

   656 

   657     def read(self, size=None):

   658         """Read data from the file.

   659         """

   660         if size is None:

   661             size = self.size - self.position

   662         else:

   663             size = min(size, self.size - self.position)

   664 

   665         if self.sparse is None:

   666             return self.readnormal(size)

   667         else:

   668             return self.readsparse(size)

   669 

   670     def readnormal(self, size):

   671         """Read operation for regular files.

   672         """

   673         self.fileobj.seek(self.offset + self.position)

   674         self.position += size

   675         return self.fileobj.read(size)

   676 

   677     def readsparse(self, size):

   678         """Read operation for sparse files.

   679         """

   680         data = []

   681         while size > 0:

   682             buf = self.readsparsesection(size)

   683             if not buf:

   684                 break

   685             size -= len(buf)

   686             data.append(buf)

   687         return "".join(data)

   688 

   689     def readsparsesection(self, size):

   690         """Read a single section of a sparse file.

   691         """

   692         section = self.sparse.find(self.position)

   693 

   694         if section is None:

   695             return ""

   696 

   697         size = min(size, section.offset + section.size - self.position)

   698 

   699         if isinstance(section, _data):

   700             realpos = section.realpos + self.position - section.offset

   701             self.fileobj.seek(self.offset + realpos)

   702             self.position += size

   703             return self.fileobj.read(size)

   704         else:

   705             self.position += size

   706             return NUL * size

   707 #class _FileInFile

   708 

   709 

   710 class ExFileObject(object):

   711     """File-like object for reading an archive member.

   712        Is returned by TarFile.extractfile().

   713     """

   714     blocksize = 1024

   715 

   716     def __init__(self, tarfile, tarinfo):

   717         self.fileobj = _FileInFile(tarfile.fileobj,

   718                                    tarinfo.offset_data,

   719                                    tarinfo.size,

   720                                    getattr(tarinfo, "sparse", None))

   721         self.name = tarinfo.name

   722         self.mode = "r"

   723         self.closed = False

   724         self.size = tarinfo.size

   725 

   726         self.position = 0

   727         self.buffer = ""

   728 

   729     def read(self, size=None):

   730         """Read at most size bytes from the file. If size is not

   731            present or None, read all data until EOF is reached.

   732         """

   733         if self.closed:

   734             raise ValueError("I/O operation on closed file")

   735 

   736         buf = ""

   737         if self.buffer:

   738             if size is None:

   739                 buf = self.buffer

   740                 self.buffer = ""

   741             else:

   742                 buf = self.buffer[:size]

   743                 self.buffer = self.buffer[size:]

   744 

   745         if size is None:

   746             buf += self.fileobj.read()

   747         else:

   748             buf += self.fileobj.read(size - len(buf))

   749 

   750         self.position += len(buf)

   751         return buf

   752 

   753     def readline(self, size=-1):

   754         """Read one entire line from the file. If size is present

   755            and non-negative, return a string with at most that

   756            size, which may be an incomplete line.

   757         """

   758         if self.closed:

   759             raise ValueError("I/O operation on closed file")

   760 

   761         if "\n" in self.buffer:

   762             pos = self.buffer.find("\n") + 1

   763         else:

   764             buffers = [self.buffer]

   765             while True:

   766                 buf = self.fileobj.read(self.blocksize)

   767                 buffers.append(buf)

   768                 if not buf or "\n" in buf:

   769                     self.buffer = "".join(buffers)

   770                     pos = self.buffer.find("\n") + 1

   771                     if pos == 0:

   772                         # no newline found.

   773                         pos = len(self.buffer)

   774                     break

   775 

   776         if size != -1:

   777             pos = min(size, pos)

   778 

   779         buf = self.buffer[:pos]

   780         self.buffer = self.buffer[pos:]

   781         self.position += len(buf)

   782         return buf

   783 

   784     def readlines(self):

   785         """Return a list with all remaining lines.

   786         """

   787         result = []

   788         while True:

   789             line = self.readline()

   790             if not line: break

   791             result.append(line)

   792         return result

   793 

   794     def tell(self):

   795         """Return the current file position.

   796         """

   797         if self.closed:

   798             raise ValueError("I/O operation on closed file")

   799 

   800         return self.position

   801 

   802     def seek(self, pos, whence=os.SEEK_SET):

   803         """Seek to a position in the file.

   804         """

   805         if self.closed:

   806             raise ValueError("I/O operation on closed file")

   807 

   808         if whence == os.SEEK_SET:

   809             self.position = min(max(pos, 0), self.size)

   810         elif whence == os.SEEK_CUR:

   811             if pos < 0:

   812                 self.position = max(self.position + pos, 0)

   813             else:

   814                 self.position = min(self.position + pos, self.size)

   815         elif whence == os.SEEK_END:

   816             self.position = max(min(self.size + pos, self.size), 0)

   817         else:

   818             raise ValueError("Invalid argument")

   819 

   820         self.buffer = ""

   821         self.fileobj.seek(self.position)

   822 

   823     def close(self):

   824         """Close the file object.

   825         """

   826         self.closed = True

   827 

   828     def __iter__(self):

   829         """Get an iterator over the file's lines.

   830         """

   831         while True:

   832             line = self.readline()

   833             if not line:

   834                 break

   835             yield line

   836 #class ExFileObject

   837 

   838 #------------------

   839 # Exported Classes

   840 #------------------

   841 class TarInfo(object):

   842     """Informational class which holds the details about an

   843        archive member given by a tar header block.

   844        TarInfo objects are returned by TarFile.getmember(),

   845        TarFile.getmembers() and TarFile.gettarinfo() and are

   846        usually created internally.

   847     """

   848 

   849     def __init__(self, name=""):

   850         """Construct a TarInfo object. name is the optional name

   851            of the member.

   852         """

   853         self.name = name        # member name (dirnames must end with '/')

   854         self.mode = 0666        # file permissions

   855         self.uid = 0            # user id

   856         self.gid = 0            # group id

   857         self.size = 0           # file size

   858         self.mtime = 0          # modification time

   859         self.chksum = 0         # header checksum

   860         self.type = REGTYPE     # member type

   861         self.linkname = ""      # link name

   862         self.uname = "user"     # user name

   863         self.gname = "group"    # group name

   864         self.devmajor = 0       # device major number

   865         self.devminor = 0       # device minor number

   866 

   867         self.offset = 0         # the tar header starts here

   868         self.offset_data = 0    # the file's data starts here

   869 

   870     def __repr__(self):

   871         return "<%s %r at %#x>" % (self.__class__.__name__,self.name,id(self))

   872 

   873     @classmethod

   874     def frombuf(cls, buf):

   875         """Construct a TarInfo object from a 512 byte string buffer.

   876         """

   877         if len(buf) != BLOCKSIZE:

   878             raise ValueError("truncated header")

   879         if buf.count(NUL) == BLOCKSIZE:

   880             raise ValueError("empty header")

   881 

   882         tarinfo = cls()

   883         tarinfo.buf = buf

   884         tarinfo.name = nts(buf[0:100])

   885         tarinfo.mode = nti(buf[100:108])

   886         tarinfo.uid = nti(buf[108:116])

   887         tarinfo.gid = nti(buf[116:124])

   888         tarinfo.size = nti(buf[124:136])

   889         tarinfo.mtime = nti(buf[136:148])

   890         tarinfo.chksum = nti(buf[148:156])

   891         tarinfo.type = buf[156:157]

   892         tarinfo.linkname = nts(buf[157:257])

   893         tarinfo.uname = nts(buf[265:297])

   894         tarinfo.gname = nts(buf[297:329])

   895         tarinfo.devmajor = nti(buf[329:337])

   896         tarinfo.devminor = nti(buf[337:345])

   897         prefix = nts(buf[345:500])

   898 

   899         if prefix and not tarinfo.issparse():

   900             tarinfo.name = prefix + "/" + tarinfo.name

   901 

   902         if tarinfo.chksum not in calc_chksums(buf):

   903             raise ValueError("invalid header")

   904         return tarinfo

   905 

   906     def tobuf(self, posix=False):

   907         """Return a tar header as a string of 512 byte blocks.

   908         """

   909         buf = ""

   910         type = self.type

   911         prefix = ""

   912 

   913         if self.name.endswith("/"):

   914             type = DIRTYPE

   915 

   916         if type in (GNUTYPE_LONGNAME, GNUTYPE_LONGLINK):

   917             # Prevent "././@LongLink" from being normalized.

   918             name = self.name

   919         else:

   920             name = normpath(self.name)

   921 

   922         if type == DIRTYPE:

   923             # directories should end with '/'

   924             name += "/"

   925 

   926         linkname = self.linkname

   927         if linkname:

   928             # if linkname is empty we end up with a '.'

   929             linkname = normpath(linkname)

   930 

   931         if posix:

   932             if self.size > MAXSIZE_MEMBER:

   933                 raise ValueError("file is too large (>= 8 GB)")

   934 

   935             if len(self.linkname) > LENGTH_LINK:

   936                 raise ValueError("linkname is too long (>%d)" % (LENGTH_LINK))

   937 

   938             if len(name) > LENGTH_NAME:

   939                 prefix = name[:LENGTH_PREFIX + 1]

   940                 while prefix and prefix[-1] != "/":

   941                     prefix = prefix[:-1]

   942 

   943                 name = name[len(prefix):]

   944                 prefix = prefix[:-1]

   945 

   946                 if not prefix or len(name) > LENGTH_NAME:

   947                     raise ValueError("name is too long")

   948 

   949         else:

   950             if len(self.linkname) > LENGTH_LINK:

   951                 buf += self._create_gnulong(self.linkname, GNUTYPE_LONGLINK)

   952 

   953             if len(name) > LENGTH_NAME:

   954                 buf += self._create_gnulong(name, GNUTYPE_LONGNAME)

   955 

   956         parts = [

   957             stn(name, 100),

   958             itn(self.mode & 07777, 8, posix),

   959             itn(self.uid, 8, posix),

   960             itn(self.gid, 8, posix),

   961             itn(self.size, 12, posix),

   962             itn(self.mtime, 12, posix),

   963             "        ", # checksum field

   964             type,

   965             stn(self.linkname, 100),

   966             stn(MAGIC, 6),

   967             stn(VERSION, 2),

   968             stn(self.uname, 32),

   969             stn(self.gname, 32),

   970             itn(self.devmajor, 8, posix),

   971             itn(self.devminor, 8, posix),

   972             stn(prefix, 155)

   973         ]

   974 

   975         buf += "".join(parts).ljust(BLOCKSIZE, NUL)

   976         chksum = calc_chksums(buf[-BLOCKSIZE:])[0]

   977         buf = buf[:-364] + "%06o\0" % chksum + buf[-357:]

   978         self.buf = buf

   979         return buf

   980 

   981     def _create_gnulong(self, name, type):

   982         """Create a GNU longname/longlink header from name.

   983            It consists of an extended tar header, with the length

   984            of the longname as size, followed by data blocks,

   985            which contain the longname as a null terminated string.

   986         """

   987         name += NUL

   988 

   989         tarinfo = self.__class__()

   990         tarinfo.name = "././@LongLink"

   991         tarinfo.type = type

   992         tarinfo.mode = 0

   993         tarinfo.size = len(name)

   994 

   995         # create extended header

   996         buf = tarinfo.tobuf()

   997         # create name blocks

   998         buf += name

   999         blocks, remainder = divmod(len(name), BLOCKSIZE)

  1000         if remainder > 0:

  1001             buf += (BLOCKSIZE - remainder) * NUL

  1002         return buf

  1003 

  1004     def isreg(self):

  1005         return self.type in REGULAR_TYPES

  1006     def isfile(self):

  1007         return self.isreg()

  1008     def isdir(self):

  1009         return self.type == DIRTYPE

  1010     def issym(self):

  1011         return self.type == SYMTYPE

  1012     def islnk(self):

  1013         return self.type == LNKTYPE

  1014     def ischr(self):

  1015         return self.type == CHRTYPE

  1016     def isblk(self):

  1017         return self.type == BLKTYPE

  1018     def isfifo(self):

  1019         return self.type == FIFOTYPE

  1020     def issparse(self):

  1021         return self.type == GNUTYPE_SPARSE

  1022     def isdev(self):

  1023         return self.type in (CHRTYPE, BLKTYPE, FIFOTYPE)

  1024 # class TarInfo

  1025 

  1026 class TarFile(object):

  1027     """The TarFile Class provides an interface to tar archives.

  1028     """

  1029 

  1030     debug = 0                   # May be set from 0 (no msgs) to 3 (all msgs)

  1031 

  1032     dereference = False         # If true, add content of linked file to the

  1033                                 # tar file, else the link.

  1034 

  1035     ignore_zeros = False        # If true, skips empty or invalid blocks and

  1036                                 # continues processing.

  1037 

  1038     errorlevel = 0              # If 0, fatal errors only appear in debug

  1039                                 # messages (if debug >= 0). If > 0, errors

  1040                                 # are passed to the caller as exceptions.

  1041 

  1042     posix = False               # If True, generates POSIX.1-1990-compliant

  1043                                 # archives (no GNU extensions!)

  1044 

  1045     fileobject = ExFileObject

  1046 

  1047     def __init__(self, name=None, mode="r", fileobj=None):

  1048         """Open an (uncompressed) tar archive `name'. `mode' is either 'r' to

  1049            read from an existing archive, 'a' to append data to an existing

  1050            file or 'w' to create a new file overwriting an existing one. `mode'

  1051            defaults to 'r'.

  1052            If `fileobj' is given, it is used for reading or writing data. If it

  1053            can be determined, `mode' is overridden by `fileobj's mode.

  1054            `fileobj' is not closed, when TarFile is closed.

  1055         """

  1056         if len(mode) > 1 or mode not in "raw":

  1057             raise ValueError("mode must be 'r', 'a' or 'w'")

  1058         self._mode = mode

  1059         self.mode = {"r": "rb", "a": "r+b", "w": "wb"}[mode]

  1060 

  1061         if not fileobj:

  1062             fileobj = file(name, self.mode)

  1063             self._extfileobj = False

  1064         else:

  1065             if name is None and hasattr(fileobj, "name"):

  1066                 name = fileobj.name

  1067             if hasattr(fileobj, "mode"):

  1068                 self.mode = fileobj.mode

  1069             self._extfileobj = True

  1070         self.name = os.path.abspath(name) if name else None

  1071         self.fileobj = fileobj

  1072 

  1073         # Init datastructures

  1074         self.closed = False

  1075         self.members = []       # list of members as TarInfo objects

  1076         self._loaded = False    # flag if all members have been read

  1077         self.offset = self.fileobj.tell()

  1078                                 # current position in the archive file

  1079         self.inodes = {}        # dictionary caching the inodes of

  1080                                 # archive members already added

  1081 

  1082         if self._mode == "r":

  1083             self.firstmember = None

  1084             self.firstmember = self.next()

  1085 

  1086         if self._mode == "a":

  1087             # Move to the end of the archive,

  1088             # before the first empty block.

  1089             self.firstmember = None

  1090             while True:

  1091                 try:

  1092                     tarinfo = self.next()

  1093                 except ReadError:

  1094                     self.fileobj.seek(0)

  1095                     break

  1096                 if tarinfo is None:

  1097                     self.fileobj.seek(- BLOCKSIZE, 1)

  1098                     break

  1099 

  1100         if self._mode in "aw":

  1101             self._loaded = True

  1102 

  1103     #--------------------------------------------------------------------------

  1104     # Below are the classmethods which act as alternate constructors to the

  1105     # TarFile class. The open() method is the only one that is needed for

  1106     # public use; it is the "super"-constructor and is able to select an

  1107     # adequate "sub"-constructor for a particular compression using the mapping

  1108     # from OPEN_METH.

  1109     #

  1110     # This concept allows one to subclass TarFile without losing the comfort of

  1111     # the super-constructor. A sub-constructor is registered and made available

  1112     # by adding it to the mapping in OPEN_METH.

  1113 

  1114     @classmethod

  1115     def open(cls, name=None, mode="r", fileobj=None, bufsize=20*512):

  1116         """Open a tar archive for reading, writing or appending. Return

  1117            an appropriate TarFile class.

  1118 

  1119            mode:

  1120            'r' or 'r:*' open for reading with transparent compression

  1121            'r:'         open for reading exclusively uncompressed

  1122            'r:gz'       open for reading with gzip compression

  1123            'r:bz2'      open for reading with bzip2 compression

  1124            'a' or 'a:'  open for appending

  1125            'w' or 'w:'  open for writing without compression

  1126            'w:gz'       open for writing with gzip compression

  1127            'w:bz2'      open for writing with bzip2 compression

  1128 

  1129            'r|*'        open a stream of tar blocks with transparent compression

  1130            'r|'         open an uncompressed stream of tar blocks for reading

  1131            'r|gz'       open a gzip compressed stream of tar blocks

  1132            'r|bz2'      open a bzip2 compressed stream of tar blocks

  1133            'w|'         open an uncompressed stream for writing

  1134            'w|gz'       open a gzip compressed stream for writing

  1135            'w|bz2'      open a bzip2 compressed stream for writing