1 # Copyright (c) 2009, Google Inc. All rights reserved.

     2 # Copyright (c) 2009 Apple Inc. All rights reserved.

     3 #

     4 # Redistribution and use in source and binary forms, with or without

     5 # modification, are permitted provided that the following conditions are

     6 # met:

     7 #

     8 #     * Redistributions of source code must retain the above copyright

     9 # notice, this list of conditions and the following disclaimer.

    10 #     * Redistributions in binary form must reproduce the above

    11 # copyright notice, this list of conditions and the following disclaimer

    12 # in the documentation and/or other materials provided with the

    13 # distribution.

    14 #     * Neither the name of Google Inc. nor the names of its

    15 # contributors may be used to endorse or promote products derived from

    16 # this software without specific prior written permission.

    17 #

    18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

    19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

    20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

    21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

    22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

    23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

    24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

    25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

    26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

    27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

    28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

    29 

    30 try:

    31     # This API exists only in Python 2.6 and higher.  :(

    32     import multiprocessing

    33 except ImportError:

    34     multiprocessing = None

    35 

    36 import errno

    37 import logging

    38 import os

    39 import platform

    40 import StringIO

    41 import signal

    42 import subprocess

    43 import sys

    44 import time

    45 

    46 from webkitpy.common.system.deprecated_logging import tee

    47 

    48 

    49 _log = logging.getLogger("webkitpy.common.system")

    50 

    51 

    52 class ScriptError(Exception):

    53 

    54     def __init__(self,

    55                  message=None,

    56                  script_args=None,

    57                  exit_code=None,

    58                  output=None,

    59                  cwd=None):

    60         if not message:

    61             message = 'Failed to run "%s"' % script_args

    62             if exit_code:

    63                 message += " exit_code: %d" % exit_code

    64             if cwd:

    65                 message += " cwd: %s" % cwd

    66 

    67         Exception.__init__(self, message)

    68         self.script_args = script_args # 'args' is already used by Exception

    69         self.exit_code = exit_code

    70         self.output = output

    71         self.cwd = cwd

    72 

    73     def message_with_output(self, output_limit=500):

    74         if self.output:

    75             if output_limit and len(self.output) > output_limit:

    76                 return "%s\nLast %s characters of output:\n%s" % \

    77                     (self, output_limit, self.output[-output_limit:])

    78             return "%s\n%s" % (self, self.output)

    79         return str(self)

    80 

    81     def command_name(self):

    82         command_path = self.script_args

    83         if type(command_path) is list:

    84             command_path = command_path[0]

    85         return os.path.basename(command_path)

    86 

    87 

    88 def run_command(*args, **kwargs):

    89     # FIXME: This should not be a global static.

    90     # New code should use Executive.run_command directly instead

    91     return Executive().run_command(*args, **kwargs)

    92 

    93 

    94 class Executive(object):

    95 

    96     def _should_close_fds(self):

    97         # We need to pass close_fds=True to work around Python bug #2320

    98         # (otherwise we can hang when we kill DumpRenderTree when we are running

    99         # multiple threads). See http://bugs.python.org/issue2320 .

   100         # Note that close_fds isn't supported on Windows, but this bug only

   101         # shows up on Mac and Linux.

   102         return sys.platform not in ('win32', 'cygwin')

   103 

   104     def _run_command_with_teed_output(self, args, teed_output):

   105         args = map(unicode, args)  # Popen will throw an exception if args are non-strings (like int())

   106         child_process = subprocess.Popen(args,

   107                                          stdout=subprocess.PIPE,

   108                                          stderr=subprocess.STDOUT,

   109                                          close_fds=self._should_close_fds())

   110 

   111         # Use our own custom wait loop because Popen ignores a tee'd

   112         # stderr/stdout.

   113         # FIXME: This could be improved not to flatten output to stdout.

   114         while True:

   115             output_line = child_process.stdout.readline()

   116             if output_line == "" and child_process.poll() != None:

   117                 # poll() is not threadsafe and can throw OSError due to:

   118                 # http://bugs.python.org/issue1731717

   119                 return child_process.poll()

   120             # We assume that the child process wrote to us in utf-8,

   121             # so no re-encoding is necessary before writing here.

   122             teed_output.write(output_line)

   123 

   124     # FIXME: Remove this deprecated method and move callers to run_command.

   125     # FIXME: This method is a hack to allow running command which both

   126     # capture their output and print out to stdin.  Useful for things

   127     # like "build-webkit" where we want to display to the user that we're building

   128     # but still have the output to stuff into a log file.

   129     def run_and_throw_if_fail(self, args, quiet=False, decode_output=True):

   130         # Cache the child's output locally so it can be used for error reports.

   131         child_out_file = StringIO.StringIO()

   132         tee_stdout = sys.stdout

   133         if quiet:

   134             dev_null = open(os.devnull, "w")  # FIXME: Does this need an encoding?

   135             tee_stdout = dev_null

   136         child_stdout = tee(child_out_file, tee_stdout)

   137         exit_code = self._run_command_with_teed_output(args, child_stdout)

   138         if quiet:

   139             dev_null.close()

   140 

   141         child_output = child_out_file.getvalue()

   142         child_out_file.close()

   143 

   144         # We assume the child process output utf-8

   145         if decode_output:

   146             child_output = child_output.decode("utf-8")

   147 

   148         if exit_code:

   149             raise ScriptError(script_args=args,

   150                               exit_code=exit_code,

   151                               output=child_output)

   152         return child_output

   153 

   154     def cpu_count(self):

   155         if multiprocessing:

   156             return multiprocessing.cpu_count()

   157         # Darn.  We don't have the multiprocessing package.

   158         system_name = platform.system()

   159         if system_name == "Darwin":

   160             return int(self.run_command(["sysctl", "-n", "hw.ncpu"]))

   161         elif system_name == "Windows":

   162             return int(os.environ.get('NUMBER_OF_PROCESSORS', 1))

   163         elif system_name == "Linux":

   164             num_cores = os.sysconf("SC_NPROCESSORS_ONLN")

   165             if isinstance(num_cores, int) and num_cores > 0:

   166                 return num_cores

   167         # This quantity is a lie but probably a reasonable guess for modern

   168         # machines.

   169         return 2

   170 

   171     def kill_process(self, pid):

   172         """Attempts to kill the given pid.

   173         Will fail silently if pid does not exist or insufficient permisssions."""

   174         if sys.platform == "win32":

   175             # We only use taskkill.exe on windows (not cygwin) because subprocess.pid

   176             # is a CYGWIN pid and taskkill.exe expects a windows pid.

   177             # Thankfully os.kill on CYGWIN handles either pid type.

   178             command = ["taskkill.exe", "/f", "/pid", pid]

   179             # taskkill will exit 128 if the process is not found.  We should log.

   180             self.run_command(command, error_handler=self.ignore_error)

   181             return

   182 

   183         # According to http://docs.python.org/library/os.html

   184         # os.kill isn't available on Windows. python 2.5.5 os.kill appears

   185         # to work in cygwin, however it occasionally raises EAGAIN.

   186         retries_left = 10 if sys.platform == "cygwin" else 1

   187         while retries_left > 0:

   188             try:

   189                 retries_left -= 1

   190                 os.kill(pid, signal.SIGKILL)

   191             except OSError, e:

   192                 if e.errno == errno.EAGAIN:

   193                     if retries_left <= 0:

   194                         _log.warn("Failed to kill pid %s.  Too many EAGAIN errors." % pid)

   195                     continue

   196                 if e.errno == errno.ESRCH:  # The process does not exist.

   197                     _log.warn("Called kill_process with a non-existant pid %s" % pid)

   198                     return

   199                 raise

   200 

   201     def _windows_image_name(self, process_name):

   202         name, extension = os.path.splitext(process_name)

   203         if not extension:

   204             # taskkill expects processes to end in .exe

   205             # If necessary we could add a flag to disable appending .exe.

   206             process_name = "%s.exe" % name

   207         return process_name

   208 

   209     def kill_all(self, process_name):

   210         """Attempts to kill processes matching process_name.

   211         Will fail silently if no process are found."""

   212         if sys.platform in ("win32", "cygwin"):

   213             image_name = self._windows_image_name(process_name)

   214             command = ["taskkill.exe", "/f", "/im", image_name]

   215             # taskkill will exit 128 if the process is not found.  We should log.

   216             self.run_command(command, error_handler=self.ignore_error)

   217             return

   218 

   219         # FIXME: This is inconsistent that kill_all uses TERM and kill_process

   220         # uses KILL.  Windows is always using /f (which seems like -KILL).

   221         # We should pick one mode, or add support for switching between them.

   222         # Note: Mac OS X 10.6 requires -SIGNALNAME before -u USER

   223         command = ["killall", "-TERM", "-u", os.getenv("USER"), process_name]

   224         # killall returns 1 if no process can be found and 2 on command error.

   225         # FIXME: We should pass a custom error_handler to allow only exit_code 1.

   226         # We should log in exit_code == 1

   227         self.run_command(command, error_handler=self.ignore_error)

   228 

   229     # Error handlers do not need to be static methods once all callers are

   230     # updated to use an Executive object.

   231 

   232     @staticmethod

   233     def default_error_handler(error):

   234         raise error

   235 

   236     @staticmethod

   237     def ignore_error(error):

   238         pass

   239 

   240     def _compute_stdin(self, input):

   241         """Returns (stdin, string_to_communicate)"""

   242         # FIXME: We should be returning /dev/null for stdin

   243         # or closing stdin after process creation to prevent

   244         # child processes from getting input from the user.

   245         if not input:

   246             return (None, None)

   247         if hasattr(input, "read"):  # Check if the input is a file.

   248             return (input, None)  # Assume the file is in the right encoding.

   249 

   250         # Popen in Python 2.5 and before does not automatically encode unicode objects.

   251         # http://bugs.python.org/issue5290

   252         # See https://bugs.webkit.org/show_bug.cgi?id=37528

   253         # for an example of a regresion caused by passing a unicode string directly.

   254         # FIXME: We may need to encode differently on different platforms.

   255         if isinstance(input, unicode):

   256             input = input.encode("utf-8")

   257         return (subprocess.PIPE, input)

   258 

   259     def _command_for_printing(self, args):

   260         """Returns a print-ready string representing command args.

   261         The string should be copy/paste ready for execution in a shell."""

   262         escaped_args = []

   263         for arg in args:

   264             if isinstance(arg, unicode):

   265                 # Escape any non-ascii characters for easy copy/paste

   266                 arg = arg.encode("unicode_escape")

   267             # FIXME: Do we need to fix quotes here?

   268             escaped_args.append(arg)

   269         return " ".join(escaped_args)

   270 

   271     # FIXME: run_and_throw_if_fail should be merged into this method.

   272     def run_command(self,

   273                     args,

   274                     cwd=None,

   275                     input=None,

   276                     error_handler=None,

   277                     return_exit_code=False,

   278                     return_stderr=True,

   279                     decode_output=True):

   280         """Popen wrapper for convenience and to work around python bugs."""

   281         assert(isinstance(args, list) or isinstance(args, tuple))

   282         start_time = time.time()

   283         args = map(unicode, args)  # Popen will throw an exception if args are non-strings (like int())

   284         stdin, string_to_communicate = self._compute_stdin(input)

   285         stderr = subprocess.STDOUT if return_stderr else None

   286 

   287         process = subprocess.Popen(args,

   288                                    stdin=stdin,

   289                                    stdout=subprocess.PIPE,

   290                                    stderr=stderr,

   291                                    cwd=cwd,

   292                                    close_fds=self._should_close_fds())

   293         output = process.communicate(string_to_communicate)[0]

   294         # run_command automatically decodes to unicode() unless explicitly told not to.

   295         if decode_output:

   296             output = output.decode("utf-8")

   297         # wait() is not threadsafe and can throw OSError due to:

   298         # http://bugs.python.org/issue1731717

   299         exit_code = process.wait()

   300 

   301         _log.debug('"%s" took %.2fs' % (self._command_for_printing(args), time.time() - start_time))

   302 

   303         if return_exit_code:

   304             return exit_code

   305 

   306         if exit_code:

   307             script_error = ScriptError(script_args=args,

   308                                        exit_code=exit_code,

   309                                        output=output,

   310                                        cwd=cwd)

   311             (error_handler or self.default_error_handler)(script_error)

   312         return output