# Copyright (c) 2009 Google Inc. All rights reserved.

# Copyright (c) 2009 Apple Inc. All rights reserved.

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions are

# met:

# 

#     * Redistributions of source code must retain the above copyright

# notice, this list of conditions and the following disclaimer.

#     * Redistributions in binary form must reproduce the above

# copyright notice, this list of conditions and the following disclaimer

# in the documentation and/or other materials provided with the

# distribution.

#     * Neither the name of Google Inc. nor the names of its

# contributors may be used to endorse or promote products derived from

# this software without specific prior written permission.

# 

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR

# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT

# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,

# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY

# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import traceback

import os

from datetime import datetime

from optparse import make_option

from StringIO import StringIO

from webkitpy.common.net.bugzilla import CommitterValidator

from webkitpy.common.net.statusserver import StatusServer

from webkitpy.common.system.executive import ScriptError

from webkitpy.common.system.deprecated_logging import error, log

from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler

from webkitpy.tool.bot.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate

from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate

from webkitpy.tool.grammar import pluralize

from webkitpy.tool.multicommandtool import Command, TryAgain

class AbstractQueue(Command, QueueEngineDelegate):

    watchers = [

    ]

    _pass_status = "Pass"

    _fail_status = "Fail"

    _error_status = "Error"

    def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations

        options_list = (options or []) + [

            make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue.  Dangerous!"),

            make_option("--exit-after-iteration", action="store", type="int", dest="iterations", default=None, help="Stop running the queue after iterating this number of times."),

        ]

        Command.__init__(self, "Run the %s" % self.name, options=options_list)

        self._iteration_count = 0

    def _cc_watchers(self, bug_id):

        try:

            self.tool.bugs.add_cc_to_bug(bug_id, self.watchers)

        except Exception, e:

            traceback.print_exc()

            log("Failed to CC watchers.")

    def run_webkit_patch(self, args):

        webkit_patch_args = [self.tool.path()]

        # FIXME: This is a hack, we should have a more general way to pass global options.

        # FIXME: We must always pass global options and their value in one argument

        # because our global option code looks for the first argument which does

        # not begin with "-" and assumes that is the command name.

        webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host]

        webkit_patch_args.extend(args)

        return self.tool.executive.run_and_throw_if_fail(webkit_patch_args)

    def _log_directory(self):

        return "%s-logs" % self.name

    # QueueEngineDelegate methods

    def queue_log_path(self):

        return os.path.join(self._log_directory(), "%s.log" % self.name)

    def work_item_log_path(self, work_item):

        raise NotImplementedError, "subclasses must implement"

    def begin_work_queue(self):

        log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root))

        if self.options.confirm:

            response = self.tool.user.prompt("Are you sure?  Type \"yes\" to continue: ")

            if (response != "yes"):

                error("User declined.")

        log("Running WebKit %s." % self.name)

    def should_continue_work_queue(self):

        self._iteration_count += 1

        return not self.options.iterations or self._iteration_count <= self.options.iterations

    def next_work_item(self):

        raise NotImplementedError, "subclasses must implement"

    def should_proceed_with_work_item(self, work_item):

        raise NotImplementedError, "subclasses must implement"

    def process_work_item(self, work_item):

        raise NotImplementedError, "subclasses must implement"

    def handle_unexpected_error(self, work_item, message):

        raise NotImplementedError, "subclasses must implement"

    # Command methods

    def execute(self, options, args, tool, engine=QueueEngine):

        self.options = options

        self.tool = tool

        return engine(self.name, self, self.tool.wakeup_event).run()

    @classmethod

    def _update_status_for_script_error(cls, tool, state, script_error, is_error=False):

        message = str(script_error)

        if is_error:

            message = "Error: %s" % message

        output = script_error.message_with_output(output_limit=1024*1024) # 1MB

        # We pre-encode the string to a byte array before passing it

        # to status_server, because ClientForm (part of mechanize)

        # wants a file-like object with pre-encoded data.

        return tool.status_server.update_status(cls.name, message, state["patch"], StringIO(output.encode("utf-8")))

class AbstractPatchQueue(AbstractQueue):

    def _update_status(self, message, patch=None, results_file=None):

        self.tool.status_server.update_status(self.name, message, patch, results_file)

    def _update_work_items(self, patch_ids):

        self.tool.status_server.update_work_items(self.name, patch_ids)

    def _did_pass(self, patch):

        self._update_status(self._pass_status, patch)

    def _did_fail(self, patch):

        self._update_status(self._fail_status, patch)

    def _did_error(self, patch, reason):

        message = "%s: %s" % (self._error_status, reason)

        self._update_status(message, patch)

    def work_item_log_path(self, patch):

        return os.path.join(self._log_directory(), "%s.log" % patch.bug_id())

    def log_progress(self, patch_ids):

        log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids))))

class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler):

    name = "commit-queue"

    def __init__(self):

        AbstractPatchQueue.__init__(self)

    # AbstractPatchQueue methods

    def begin_work_queue(self):

        AbstractPatchQueue.begin_work_queue(self)

        self.committer_validator = CommitterValidator(self.tool.bugs)

    def _validate_patches_in_commit_queue(self):

        # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers.

        bug_ids = self.tool.bugs.queries.fetch_bug_ids_from_commit_queue()

        all_patches = sum([self.tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True) for bug_id in bug_ids], [])

        return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches)

    def _patch_cmp(self, a, b):

        # Sort first by is_rollout, then by attach_date.

        # Reversing the order so that is_rollout is first.

        rollout_cmp = cmp(b.is_rollout(), a.is_rollout())

        if (rollout_cmp != 0):

            return rollout_cmp

        return cmp(a.attach_date(), b.attach_date())

    def next_work_item(self):

        patches = self._validate_patches_in_commit_queue()

        patches = sorted(patches, self._patch_cmp)

        self._update_work_items([patch.id() for patch in patches])

        if not patches:

            self._update_status("Empty queue")

            return None

        # Only bother logging if we have patches in the queue.

        self.log_progress([patch.id() for patch in patches])

        return patches[0]

    def _can_build_and_test(self):

        try:

            self.run_webkit_patch([

                "build-and-test",

                "--force-clean",

                "--build",

                "--test",

                "--non-interactive",

                "--no-update",

                "--build-style=both",

                "--quiet"])

        except ScriptError, e:

            self._update_status("Unable to successfully build and test", None)

            return False

        return True

    def should_proceed_with_work_item(self, patch):

        patch_text = "rollout patch" if patch.is_rollout() else "patch"

        self._update_status("Landing %s" % patch_text, patch)

        return True

    def _land(self, patch, first_run=False):

        try:

            args = [

                "land-attachment",

                "--force-clean",

                "--build",

                "--non-interactive",

                "--ignore-builders",

                "--build-style=both",

                "--quiet",

                patch.id()

            ]

            # We don't bother to run tests for rollouts as that makes them too slow.

            if not patch.is_rollout():

                args.append("--test")

            if not first_run:

                # The first time through, we don't reject the patch from the

                # commit queue because we want to make sure we can build and

                # test ourselves. However, the second time through, we

                # register ourselves as the parent-command so we can reject

                # the patch on failure.

                args.append("--parent-command=commit-queue")

                # The second time through, we also don't want to update so we

                # know we're testing the same revision that we successfully

                # built and tested.

                args.append("--no-update")

            self.run_webkit_patch(args)

            self._did_pass(patch)

            return True

        except ScriptError, e:

            if first_run:

                return False

            self._did_fail(patch)

            raise

    def process_work_item(self, patch):

        self._cc_watchers(patch.bug_id())

        if not self._land(patch, first_run=True):

            # The patch failed to land, but the bots were green. It's possible

            # that the bots were behind. To check that case, we try to build and

            # test ourselves.

            if not self._can_build_and_test():

                return False

            # Hum, looks like the patch is actually bad. Of course, we could

            # have been bitten by a flaky test the first time around.  We try

            # to land again.  If it fails a second time, we're pretty sure its

            # a bad test and re can reject it outright.

            self._land(patch)

        return True

    def handle_unexpected_error(self, patch, message):

        self.committer_validator.reject_patch_from_commit_queue(patch.id(), message)

    # StepSequenceErrorHandler methods

    @staticmethod

    def _error_message_for_bug(tool, status_id, script_error):

        if not script_error.output:

            return script_error.message_with_output()

        results_link = tool.status_server.results_url_for_status(status_id)

        return "%s\nFull output: %s" % (script_error.message_with_output(), results_link)

    @classmethod

    def handle_script_error(cls, tool, state, script_error):

        status_id = cls._update_status_for_script_error(tool, state, script_error)

        validator = CommitterValidator(tool.bugs)

        validator.reject_patch_from_commit_queue(state["patch"].id(), cls._error_message_for_bug(tool, status_id, script_error))

    @classmethod

    def handle_checkout_needs_update(cls, tool, state, options, error):

        # The only time when we find out that out checkout needs update is

        # when we were ready to actually pull the trigger and land the patch.

        # Rather than spinning in the master process, we retry without

        # building or testing, which is much faster.

        options.build = False

        options.test = False

        options.update = True

        raise TryAgain()

class RietveldUploadQueue(AbstractPatchQueue, StepSequenceErrorHandler):

    name = "rietveld-upload-queue"

    def __init__(self):

        AbstractPatchQueue.__init__(self)

    # AbstractPatchQueue methods

    def next_work_item(self):

        patch_id = self.tool.bugs.queries.fetch_first_patch_from_rietveld_queue()

        if patch_id:

            return patch_id

        self._update_status("Empty queue")

    def should_proceed_with_work_item(self, patch):

        self._update_status("Uploading patch", patch)

        return True

    def process_work_item(self, patch):

        try:

            self.run_webkit_patch(["post-attachment-to-rietveld", "--force-clean", "--non-interactive", "--parent-command=rietveld-upload-queue", patch.id()])

            self._did_pass(patch)

            return True

        except ScriptError, e:

            if e.exit_code != QueueEngine.handled_error_code:

                self._did_fail(patch)

            raise e

    @classmethod

    def _reject_patch(cls, tool, patch_id):

        tool.bugs.set_flag_on_attachment(patch_id, "in-rietveld", "-")

    def handle_unexpected_error(self, patch, message):

        log(message)

        self._reject_patch(self.tool, patch.id())

    # StepSequenceErrorHandler methods

    @classmethod

    def handle_script_error(cls, tool, state, script_error):

        log(script_error.message_with_output())

        cls._update_status_for_script_error(tool, state, script_error)

        cls._reject_patch(tool, state["patch"].id())

class AbstractReviewQueue(AbstractPatchQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler):

    def __init__(self, options=None):

        AbstractPatchQueue.__init__(self, options)

    def review_patch(self, patch):

        raise NotImplementedError, "subclasses must implement"

    # PersistentPatchCollectionDelegate methods

    def collection_name(self):

        return self.name

    def fetch_potential_patch_ids(self):

        return self.tool.bugs.queries.fetch_attachment_ids_from_review_queue()

    def status_server(self):

        return self.tool.status_server

    def is_terminal_status(self, status):

        return status == "Pass" or status == "Fail" or status.startswith("Error:")

    # AbstractPatchQueue methods

    def begin_work_queue(self):

        AbstractPatchQueue.begin_work_queue(self)

        self._patches = PersistentPatchCollection(self)

    def next_work_item(self):

        patch_id = self._patches.next()

        if patch_id:

            return self.tool.bugs.fetch_attachment(patch_id)

        self._update_status("Empty queue")

    def should_proceed_with_work_item(self, patch):

        raise NotImplementedError, "subclasses must implement"

    def process_work_item(self, patch):

        try:

            if not self.review_patch(patch):

                return False

            self._did_pass(patch)

            return True

        except ScriptError, e:

            if e.exit_code != QueueEngine.handled_error_code:

                self._did_fail(patch)

            raise e

    def handle_unexpected_error(self, patch, message):

        log(message)

    # StepSequenceErrorHandler methods

    @classmethod

    def handle_script_error(cls, tool, state, script_error):

        log(script_error.message_with_output())

class StyleQueue(AbstractReviewQueue):

    name = "style-queue"

    def __init__(self):

        AbstractReviewQueue.__init__(self)

    def should_proceed_with_work_item(self, patch):

        self._update_status("Checking style", patch)

        return True

    def review_patch(self, patch):

        self.run_webkit_patch(["check-style", "--force-clean", "--non-interactive", "--parent-command=style-queue", patch.id()])

        return True

    @classmethod

    def handle_script_error(cls, tool, state, script_error):

        is_svn_apply = script_error.command_name() == "svn-apply"

        status_id = cls._update_status_for_script_error(tool, state, script_error, is_error=is_svn_apply)

        if is_svn_apply:

            QueueEngine.exit_after_handled_error(script_error)

        message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (state["patch"].id(), cls.name, script_error.message_with_output(output_limit=3*1024))

        tool.bugs.post_comment_to_bug(state["patch"].bug_id(), message, cc=cls.watchers)

        exit(1)