Mini Shell

# -*- test-case-name: twisted.trial._dist.test.test_disttrial -*-
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.

"""
This module contains the trial distributed runner, the management class
responsible for coordinating all of trial's behavior at the highest level.

@since: 12.3
"""

import os
import sys

from twisted.internet.defer import DeferredList
from twisted.internet.task import cooperate
from twisted.python.filepath import FilePath
from twisted.python.modules import theSystemPath
from twisted.trial._asyncrunner import _iterateTests
from twisted.trial._dist import _WORKER_AMP_STDIN, _WORKER_AMP_STDOUT
from twisted.trial._dist.distreporter import DistReporter
from twisted.trial._dist.worker import LocalWorker, LocalWorkerAMP
from twisted.trial.reporter import UncleanWarningsReporterWrapper
from twisted.trial.util import _unusedTestDirectory


class DistTrialRunner:
    """
    A specialized runner for distributed trial. The runner launches a number of
    local worker processes which will run tests.

    @ivar _workerNumber: the number of workers to be spawned.
    @type _workerNumber: C{int}

    @ivar _stream: stream which the reporter will use.

    @ivar _reporterFactory: the reporter class to be used.
    """

    _distReporterFactory = DistReporter

    def _makeResult(self):
        """
        Make reporter factory, and wrap it with a L{DistReporter}.
        """
        reporter = self._reporterFactory(
            self._stream, self._tbformat, realtime=self._rterrors
        )
        if self._uncleanWarnings:
            reporter = UncleanWarningsReporterWrapper(reporter)
        return self._distReporterFactory(reporter)

    def __init__(
        self,
        reporterFactory,
        workerNumber,
        workerArguments,
        stream=None,
        tracebackFormat="default",
        realTimeErrors=False,
        uncleanWarnings=False,
        logfile="test.log",
        workingDirectory="_trial_temp",
    ):
        self._workerNumber = workerNumber
        self._workerArguments = workerArguments
        self._reporterFactory = reporterFactory
        if stream is None:
            stream = sys.stdout
        self._stream = stream
        self._tbformat = tracebackFormat
        self._rterrors = realTimeErrors
        self._uncleanWarnings = uncleanWarnings
        self._result = None
        self._workingDirectory = workingDirectory
        self._logFile = logfile
        self._logFileObserver = None
        self._logFileObject = None
        self._logWarnings = False

    def writeResults(self, result):
        """
        Write test run final outcome to result.

        @param result: A C{TestResult} which will print errors and the summary.
        """
        result.done()

    def createLocalWorkers(self, protocols, workingDirectory):
        """
        Create local worker protocol instances and return them.

        @param protocols: An iterable of L{LocalWorkerAMP} instances.

        @param workingDirectory: The base path in which we should run the
            workers.
        @type workingDirectory: C{str}

        @return: A list of C{quantity} C{LocalWorker} instances.
        """
        return [
            LocalWorker(protocol, os.path.join(workingDirectory, str(x)), self._logFile)
            for x, protocol in enumerate(protocols)
        ]

    def launchWorkerProcesses(self, spawner, protocols, arguments):
        """
        Spawn processes from a list of process protocols.

        @param spawner: A C{IReactorProcess.spawnProcess} implementation.

        @param protocols: An iterable of C{ProcessProtocol} instances.

        @param arguments: Extra arguments passed to the processes.
        """
        workertrialPath = theSystemPath["twisted.trial._dist.workertrial"].filePath.path
        childFDs = {
            0: "w",
            1: "r",
            2: "r",
            _WORKER_AMP_STDIN: "w",
            _WORKER_AMP_STDOUT: "r",
        }
        environ = os.environ.copy()
        # Add an environment variable containing the raw sys.path, to be used
        # by subprocesses to try to make it identical to the parent's.
        environ["PYTHONPATH"] = os.pathsep.join(sys.path)
        for worker in protocols:
            args = [sys.executable, workertrialPath]
            args.extend(arguments)
            spawner(worker, sys.executable, args=args, childFDs=childFDs, env=environ)

    def _driveWorker(self, worker, result, testCases, cooperate):
        """
        Drive a L{LocalWorkerAMP} instance, iterating the tests and calling
        C{run} for every one of them.

        @param worker: The L{LocalWorkerAMP} to drive.

        @param result: The global L{DistReporter} instance.

        @param testCases: The global list of tests to iterate.

        @param cooperate: The cooperate function to use, to be customized in
            tests.
        @type cooperate: C{function}

        @return: A C{Deferred} firing when all the tests are finished.
        """

        def resultErrback(error, case):
            result.original.addFailure(case, error)
            return error

        def task(case):
            d = worker.run(case, result)
            d.addErrback(resultErrback, case)
            return d

        return cooperate(task(case) for case in testCases).whenDone()

    def run(self, suite, reactor=None, cooperate=cooperate, untilFailure=False):
        """
        Spawn local worker processes and load tests. After that, run them.

        @param suite: A tests suite to be run.

        @param reactor: The reactor to use, to be customized in tests.
        @type reactor: A provider of
            L{twisted.internet.interfaces.IReactorProcess}

        @param cooperate: The cooperate function to use, to be customized in
            tests.
        @type cooperate: C{function}

        @param untilFailure: If C{True}, continue to run the tests until they
            fail.
        @type untilFailure: C{bool}.

        @return: The test result.
        @rtype: L{DistReporter}
        """
        if reactor is None:
            from twisted.internet import reactor
        result = self._makeResult()
        count = suite.countTestCases()
        self._stream.write("Running %d tests.\n" % (count,))

        if not count:
            # Take a shortcut if there is no test
            suite.run(result.original)
            self.writeResults(result)
            return result

        testDir, testDirLock = _unusedTestDirectory(FilePath(self._workingDirectory))
        workerNumber = min(count, self._workerNumber)
        ampWorkers = [LocalWorkerAMP() for x in range(workerNumber)]
        workers = self.createLocalWorkers(ampWorkers, testDir.path)
        processEndDeferreds = [worker.endDeferred for worker in workers]
        self.launchWorkerProcesses(reactor.spawnProcess, workers, self._workerArguments)

        def runTests():
            testCases = iter(list(_iterateTests(suite)))

            workerDeferreds = []
            for worker in ampWorkers:
                workerDeferreds.append(
                    self._driveWorker(worker, result, testCases, cooperate=cooperate)
                )
            return DeferredList(
                workerDeferreds, consumeErrors=True, fireOnOneErrback=True
            )

        stopping = []

        def nextRun(ign, n):
            if untilFailure:
                self._stream.write("Test Pass %d\n" % (n,))
            self.writeResults(result)
            if not untilFailure:
                return
            if not result.wasSuccessful():
                return
            d = runTests()
            return d.addCallback(nextRun, n + 1)

        def stop(ign):
            testDirLock.unlock()
            if not stopping:
                stopping.append(None)
                reactor.stop()

        def beforeShutDown():
            if not stopping:
                stopping.append(None)
                d = DeferredList(processEndDeferreds, consumeErrors=True)
                return d.addCallback(continueShutdown)

        def continueShutdown(ign):
            self.writeResults(result)
            return ign

        d = runTests()
        d.addCallback(nextRun, 1)
        d.addBoth(stop)

        reactor.addSystemEventTrigger("before", "shutdown", beforeShutDown)
        reactor.run()

        return result

    def runUntilFailure(self, suite):
        """
        Run the tests with local worker processes until they fail.

        @param suite: A tests suite to be run.
        """
        return self.run(suite, untilFailure=True)