runner.cpp

00001 /*
00002  *
00003  * Copyright (C)  2004  Zack Rusin <zack@kde.org>
00004  * Copyright (C)  2005  Jeroen Wijnhout <Jeroen.Wijnhout@kdemail.net>
00005  *
00006  * Redistribution and use in source and binary forms, with or without
00007  * modification, are permitted provided that the following conditions
00008  * are met:
00009  *
00010  * 1. Redistributions of source code must retain the above copyright
00011  *   notice, this list of conditions and the following disclaimer.
00012  * 2. Redistributions in binary form must reproduce the above copyright
00013  *   notice, this list of conditions and the following disclaimer in the
00014  *   documentation and/or other materials provided with the distribution.
00015  *
00016  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
00017  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00018  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
00019  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
00020  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
00021  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
00022  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
00023  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
00024  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
00025  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00026  */
00027 
00028 #include <stdio.h>
00029 #include <iostream>
00030 using namespace std;
00031 
00032 #include <qregexp.h>
00033 #include <qdir.h>
00034 #include <qmetaobject.h>
00035 
00036 #include <kdebug.h>
00037 #include <klibloader.h>
00038 #include <kglobal.h>
00039 #include <kstandarddirs.h>
00040 
00041 #include "runner.h"
00042 #include "tester.h"
00043 
00044 namespace KUnitTest
00045 {
00046     Runner *Runner::s_self = 0L;
00047     bool    Runner::s_debugCapturingEnabled = false;
00048 
00049     void Runner::registerTester(const char *name, Tester *test)
00050     {
00051         Runner::self()->m_registry.insert(name, test);
00052     }
00053 
00054     void Runner::loadModules(const QString &folder, const QString &query)
00055     {
00056         QRegExp reQuery(query);
00057         QDir dir(folder, "kunittest_*.la");
00058 
00059         // Add the folder to the "module" resource such that the KLibLoader can
00060         // find the modules in this folder.
00061         KGlobal::dirs()->addResourceDir("module", folder);
00062         kdDebug() << "Looking in folder: " << dir.absPath() << endl;
00063 
00064         // Get a list of all modules.
00065         QStringList modules = dir.entryList();
00066 
00067         for ( uint i = 0; i < modules.count(); ++i )
00068         {
00069             QString module = modules[i];
00070             kdDebug() << "Module: " << dir.absPath() + "/" + module << endl;
00071 
00072             if ( reQuery.search(module) != -1 )
00073             {
00074                 // strip the .la extension
00075                 module.truncate(module.length()-3);
00076                 KLibFactory *factory = KLibLoader::self()->factory(module.local8Bit());
00077                 if ( factory )
00078                     factory->create();
00079                 else {
00080                     kdWarning() << "\tError loading " << module << " : " << KLibLoader::self()->lastErrorMessage() << endl;
00081                     ::exit( 1 );
00082                 }
00083             }
00084             else
00085                 kdDebug() << "\tModule doesn't match." << endl;
00086         }
00087     }
00088 
00089     void Runner::setDebugCapturingEnabled(bool enabled)
00090     {
00091       s_debugCapturingEnabled = enabled;
00092     }
00093 
00094     RegistryType &Runner::registry()
00095     {
00096         return m_registry;
00097     }
00098 
00099     int Runner::numberOfTestCases()
00100     {
00101         return m_registry.count();
00102     }
00103 
00104     Runner *Runner::self()
00105     {
00106         if ( s_self == 0L ) s_self = new Runner();
00107 
00108         return s_self;
00109     }
00110 
00111     Runner::Runner()
00112     {
00113         reset();
00114     }
00115 
00116     int Runner::numberOfTests() const
00117     {
00118         return globalSteps;
00119     }
00120 
00121     int Runner::numberOfPassedTests() const
00122     {
00123         return globalPasses;
00124     }
00125 
00126     int Runner::numberOfFailedTests() const
00127     {
00128         return globalFails;
00129     }
00130 
00131     int Runner::numberOfExpectedFailures() const
00132     {
00133         return globalXFails;
00134     }
00135 
00136     int Runner::numberOfSkippedTests() const
00137     {
00138         return globalSkipped;
00139     }
00140 
00141     void Runner::reset()
00142     {
00143         globalSteps = 0;
00144         globalPasses = 0;
00145         globalFails = 0;
00146         globalXFails = 0;
00147         globalXPasses = 0;
00148         globalSkipped = 0;
00149     }
00150 
00151     int Runner::runTests()
00152     {
00153         globalSteps = 0;
00154         globalPasses = 0;
00155         globalFails = 0;
00156         globalXFails = 0;
00157         globalXPasses = 0;
00158         globalSkipped = 0;
00159 
00160         cout << "# Running normal tests... #" << endl << endl;
00161         RegistryIteratorType it(m_registry);
00162 
00163         for( ; it.current(); ++it )
00164             runTest(it.currentKey());
00165 
00166 #if 0 // very thorough, but not very readable
00167         cout << "# Done with normal tests:" << endl;
00168         cout << "  Total test cases: " << m_registry.count() << endl;
00169         cout << "  Total test steps                                 : " << globalSteps << endl;
00170         cout << "    Total passed test steps (including unexpected) : " << globalPasses << endl;
00171         cout << "      Total unexpected passed test steps           :  " << globalXPasses << endl;
00172         cout << "    Total failed test steps (including expected)   :  " << globalFails << endl;
00173         cout << "      Total expected failed test steps             :  " << globalXFails << endl;
00174         cout << "    Total skipped test steps                       :  " << globalSkipped << endl;
00175 #else
00176         unsigned int numTests = m_registry.count(); // should this be globalSteps instead?
00177         QString str;
00178         if ( globalFails == 0 )
00179             if ( globalXFails == 0 )
00180                 str = QString( "All %1 tests passed" ).arg( numTests );
00181             else
00182                 str = QString( "All %1 tests behaved as expected (%2 expected failures)" ).arg( numTests ).arg( globalXFails );
00183         else
00184             if ( globalXPasses == 0 )
00185                 str = QString( "%1 of %2 tests failed" ).arg( globalFails ).arg( numTests );
00186             else
00187                 str = QString( "%1 of %2 tests did not behave as expected (%1 unexpected passes)" ).arg( globalFails ).arg( numTests ).arg( globalXPasses );
00188         if ( globalSkipped )
00189             str += QString( " (%1 tests skipped)" ).arg( globalSkipped );
00190         cout << str.local8Bit() << endl;
00191 #endif
00192 
00193         return m_registry.count();
00194     }
00195 
00196     void Runner::runMatchingTests(const QString &prefix)
00197     {
00198         RegistryIteratorType it(m_registry);
00199         for( ; it.current(); ++it )
00200             if ( QString(it.currentKey()).startsWith(prefix) )
00201                 runTest(it.currentKey());
00202     }
00203 
00204     void Runner::runTest(const char *name)
00205     {
00206         Tester *test = m_registry.find(name);
00207         if ( test == 0L ) return;
00208 
00209         if ( s_debugCapturingEnabled )
00210         {
00211           cout << "KUnitTest_Debug_Start[" << name << "]" << endl;
00212         }
00213 
00214         test->results()->clear();
00215         test->allTests();
00216 
00217         if ( s_debugCapturingEnabled )
00218         {
00219           cout << "KUnitTest_Debug_End[" << name << "]" << endl << endl << flush;
00220         }
00221 
00222         int numPass = 0;
00223         int numFail = 0;
00224         int numXFail = 0;
00225         int numXPass = 0;
00226         int numSkip = 0;
00227         QStringList xpassList;
00228         QStringList errorList;
00229         QStringList xfailList;
00230         QStringList skipList;
00231 
00232         if ( test->inherits("KUnitTest::SlotTester") )
00233         {
00234             SlotTester *sltest = static_cast<SlotTester*>(test);
00235             TestResultsListIteratorType it(sltest->resultsList());
00236             for ( ; it.current(); ++it)
00237             {
00238                 numPass += it.current()->passed() + it.current()->xpasses();
00239                 numFail += it.current()->errors() + it.current()->xfails();
00240                 numXFail += it.current()->xfails();
00241                 numXPass += it.current()->xpasses();
00242                 numSkip += it.current()->skipped();
00243                 globalSteps += it.current()->testsFinished();
00244 
00245                 xpassList += it.current()->xpassList();
00246                 errorList += it.current()->errorList();
00247                 xfailList += it.current()->xfailList();
00248                 skipList += it.current()->skipList();
00249             }
00250         }
00251         else
00252         {
00253             numPass= test->results()->passed() + test->results()->xpasses();
00254             numFail= test->results()->errors() + test->results()->xfails();
00255             numXFail = test->results()->xfails();
00256             numXPass = test->results()->xpasses();
00257             numSkip= test->results()->skipped();
00258             globalSteps += test->results()->testsFinished();
00259 
00260             xpassList += test->results()->xpassList();
00261             errorList += test->results()->errorList();
00262             xfailList += test->results()->xfailList();
00263             skipList += test->results()->skipList();
00264         }
00265 
00266 
00267         globalPasses += numPass;
00268         globalFails += numFail;
00269         globalXFails += numXFail;
00270         globalXPasses += numXPass;
00271         globalSkipped += numSkip;
00272 
00273         cout << name << " - ";
00274         cout << numPass << " test" << ( ( 1 == numPass )?"":"s") << " passed";
00275         if ( 0 < xpassList.count() ) {
00276             cout << " (" << numXPass << " unexpected pass" << ( ( 1 == numXPass )?"":"es") << ")";
00277         }
00278         cout << ", " << numFail << " test" << ( ( 1 == numFail )?"":"s") << " failed";
00279         if ( 0 < numXFail  ) {
00280             cout << " (" << numXFail << " expected failure" << ( ( 1 == numXFail )?"":"s") << ")";
00281         }
00282         if ( 0 < numSkip ) {
00283             cout << "; also " << numSkip << " skipped";
00284         }
00285         cout  << endl;
00286 
00287         if ( 0 < numXPass  ) {
00288         cout << "    Unexpected pass" << ( ( 1 == numXPass )?"":"es") << ":" << endl;
00289         QStringList list = xpassList;
00290         for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
00291             cout << "\t" << (*itr).latin1() << endl;
00292         }
00293         }
00294         if ( 0 < (numFail - numXFail) ) {
00295         cout << "    Unexpected failure" << ( ( 1 == numFail )?"":"s") << ":" << endl;
00296         QStringList list = errorList;
00297         for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
00298             cout << "\t" << (*itr).latin1() << endl;
00299         }
00300         }
00301         if ( 0 < numXFail ) {
00302         cout << "    Expected failure" << ( ( 1 == numXFail)?"":"s") << ":" << endl;
00303         QStringList list = xfailList;
00304         for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
00305             cout << "\t" << (*itr).latin1() << endl;
00306         }
00307         }
00308         if ( 0 < numSkip ) {
00309             cout << "    Skipped test" << ( ( 1 == numSkip )?"":"s") << ":" << endl;
00310             QStringList list = skipList;
00311             for ( QStringList::Iterator itr = list.begin(); itr != list.end(); ++itr ) {
00312             cout << "\t" << (*itr).latin1() << endl;
00313             }
00314         }
00315         cout << endl;
00316 
00317         emit finished(name, test);
00318     }
00319 }
00320 
00321 #include "runner.moc"
00322 
KDE Home | KDE Accessibility Home | Description of Access Keys