1 /****************************************************************************** 2 3 Flexible unittest runner 4 5 This module provides a more flexible unittest runner. 6 7 The goals for this test runner is to function as a standalone program, 8 instead of being run before another program's main(), as is common in D. 9 10 To achieve this, the main() function is provided by this module too. 11 12 To use it, just import this module and any other module you want to test, 13 for example: 14 15 --- 16 module tester; 17 import ocean.core.UnitTestRunner; 18 import mymodule; 19 --- 20 21 That's it. Compile with: dmd -unittest tester.d mymodule.d 22 23 You can control the unittest execution, try ./tester -h for help on the 24 available options. 25 26 Tester status codes: 27 28 0 - All tests passed 29 2 - Wrong command line arguments 30 4 - One or more tests failed 31 8 - One or more tests had errors (unexpected problems) 32 16 - Error writing to XML file 33 34 Status codes can be aggregated via ORing, so for example, a status of 12 35 means both failures and errors were encountered, and status of 20 means 36 failures were encountered and there was an error writing the XML file. 37 38 Copyright: 39 Copyright (c) 2009-2016 dunnhumby Germany GmbH. 40 All rights reserved. 41 42 License: 43 Boost Software License Version 1.0. See LICENSE_BOOST.txt for details. 44 Alternatively, this file may be distributed under the terms of the Tango 45 3-Clause BSD License (see LICENSE_BSD.txt for details). 46 47 *******************************************************************************/ 48 49 module ocean.core.UnitTestRunner; 50 51 import ocean.transition; 52 53 54 import ocean.core.Verify; 55 import ocean.stdc.string: strdup, strlen, strncmp; 56 import core.sys.posix.unistd: unlink; 57 import core.sys.posix.sys.time: gettimeofday, timeval; 58 59 static if (__VERSION__ >= 2000 && __VERSION__ < 2073) 60 { 61 extern(C) char* basename (char*); 62 63 void timersub (in timeval* a, in timeval* b, timeval *result) 64 { 65 result.tv_sec = a.tv_sec - b.tv_sec; 66 result.tv_usec = a.tv_usec - b.tv_usec; 67 if (result.tv_usec < 0) 68 { 69 --result.tv_sec; 70 result.tv_usec += 1_000_000; 71 } 72 } 73 } 74 else 75 { 76 import core.sys.posix.libgen: basename; 77 import core.sys.posix.sys.time: timersub; 78 } 79 80 import ocean.io.Stdout: Stdout, Stderr; 81 import ocean.io.stream.Format: FormatOutput; 82 import ocean.io.stream.TextFile: TextFileOutput; 83 import ocean.text.xml.Document: Document; 84 import ocean.text.xml.DocPrinter: DocPrinter; 85 import ocean.text.convert.Formatter; 86 import ocean.core.Test: TestException, test; 87 import ocean.core.array.Mutation: uniq; 88 import core.memory; 89 import core.runtime: Runtime; 90 91 92 /****************************************************************************** 93 94 Handle all the details about unittest execution. 95 96 ******************************************************************************/ 97 98 private scope class UnitTestRunner 99 { 100 101 /************************************************************************** 102 103 Options parsed from the command-line 104 105 ***************************************************************************/ 106 107 private cstring prog; 108 private bool help = false; 109 private size_t verbose = 0; 110 private bool summary = false; 111 private bool keep_going = false; 112 private cstring[] packages = null; 113 private cstring xml_file; 114 115 116 /************************************************************************** 117 118 Aliases for easier access to symbols with long and intricate names 119 120 ***************************************************************************/ 121 122 private alias Document!(char) XmlDoc; 123 private alias XmlDoc.Node XmlNode; /// ditto 124 125 126 /************************************************************************** 127 128 Buffer used for text conversions 129 130 ***************************************************************************/ 131 132 private mstring buf; 133 134 135 /************************************************************************** 136 137 XML document used to produce the XML test results report 138 139 ***************************************************************************/ 140 141 private XmlDoc xml_doc; 142 143 144 /************************************************************************** 145 146 Static constructor replacing the default Tango unittest runner 147 148 ***************************************************************************/ 149 150 static this ( ) 151 { 152 Runtime.moduleUnitTester(&this.dummyUnitTestRunner); 153 } 154 155 156 /************************************************************************** 157 158 Dummy unittest runner. 159 160 This runner does nothing because we handle all the unittest execution 161 directly in the main() function, so we can parse the program's argument 162 before running the unittests. 163 164 Returns: 165 true to tell the runtime we want to run main() 166 167 ***************************************************************************/ 168 169 private static bool dummyUnitTestRunner() 170 { 171 return true; 172 } 173 174 175 /************************************************************************** 176 177 Run all the unittest registered by the runtime. 178 179 The parseArgs() function must be called before this method. 180 181 Returns: 182 exit status to pass to the operating system. 183 184 ***************************************************************************/ 185 186 private int run ( ) 187 { 188 verify(prog.length > 0); 189 190 timeval start_time = this.now(); 191 192 size_t passed = 0; 193 size_t failed = 0; 194 size_t errored = 0; 195 size_t skipped = 0; 196 size_t no_tests = 0; 197 size_t no_match = 0; 198 size_t gc_usage_before, gc_usage_after, mem_free; 199 bool collect_gc_usage = !!this.verbose; 200 201 if (this.verbose) 202 Stdout.formatln("{}: unit tests started", this.prog); 203 204 foreach ( m; ModuleInfo ) 205 { 206 if (!this.shouldTest(m.name)) 207 { 208 no_match++; 209 if (this.verbose > 1) 210 Stdout.formatln("{}: {}: skipped (not in " ~ 211 "modules to test)", this.prog, m.name); 212 continue; 213 } 214 215 if (m.unitTest is null) 216 { 217 no_tests++; 218 if (this.verbose > 1) 219 Stdout.formatln("{}: {}: skipped (no unittests)", 220 this.prog, m.name); 221 this.xmlAddSkipped(m.name); 222 continue; 223 } 224 225 if ((failed || errored) && !this.keep_going) 226 { 227 skipped++; 228 if (this.verbose > 2) 229 Stdout.formatln( 230 "{}: {}: skipped (one failed and no --keep-going)", 231 this.prog, m.name); 232 this.xmlAddSkipped(m.name); 233 continue; 234 } 235 236 if (this.verbose) 237 { 238 Stdout.format("{}: {}: testing ...", this.prog, m.name).flush(); 239 } 240 241 // we have a unittest, run it 242 timeval t; 243 // XXX: We can't use this.buf because it will be messed up when 244 // calling toHumanTime() and the different xmlAdd*() methods 245 static mstring e; 246 e.length = 0; 247 enableStomping(e); 248 scope (exit) 249 { 250 if (this.verbose) 251 Stdout.newline(); 252 if (e !is null) 253 Stdout.formatln("{}", e); 254 } 255 if (collect_gc_usage) 256 { 257 GC.collect(); 258 ocean.transition.gc_usage(gc_usage_before, mem_free); 259 } 260 switch (this.timedTest(m, t, e)) 261 { 262 case Result.Pass: 263 passed++; 264 if (this.verbose) 265 { 266 ocean.transition.gc_usage(gc_usage_after, mem_free); 267 Stdout.format(" PASS [{}, {} bytes ({} -> {})]", 268 this.toHumanTime(t), 269 cast(long)(gc_usage_after - gc_usage_before), 270 gc_usage_before, gc_usage_after); 271 } 272 this.xmlAddSuccess(m.name, t); 273 continue; 274 275 case Result.Fail: 276 failed++; 277 if (this.verbose) 278 Stdout.format(" FAIL [{}]", this.toHumanTime(t)); 279 this.xmlAddFailure(m.name, t, e); 280 break; 281 282 case Result.Error: 283 errored++; 284 if (this.verbose) 285 Stdout.format(" ERROR [{}]", this.toHumanTime(t)); 286 this.xmlAddFailure!("error")(m.name, t, e); 287 break; 288 289 default: 290 assert(false); 291 } 292 293 if (!this.keep_going) 294 continue; 295 296 if (this.verbose > 2) 297 Stdout.format(" (continuing, --keep-going used)"); 298 } 299 300 timeval total_time = elapsedTime(start_time); 301 302 if (this.summary) 303 { 304 Stdout.format("{}: {} modules passed, {} failed, " 305 ~ "{} with errors, {} without unittests", 306 this.prog, passed, failed, errored, no_tests); 307 if (!this.keep_going && failed) 308 Stdout.format(", {} skipped", skipped); 309 if (this.verbose > 1) 310 Stdout.format(", {} didn't match -p/--package", no_match); 311 Stdout.formatln(" [{}]", this.toHumanTime(total_time)); 312 } 313 314 bool xml_ok = this.writeXml( 315 passed + failed + errored + no_tests + skipped, 316 no_tests + skipped, failed, errored, total_time); 317 318 int ret = 0; 319 320 if (!xml_ok) 321 ret |= 16; 322 323 if (errored) 324 ret |= 8; 325 326 if (failed) 327 ret |= 4; 328 329 return ret; 330 } 331 332 333 /************************************************************************** 334 335 Add a skipped test node to the XML document 336 337 Params: 338 name = name of the test to add to the XML document 339 340 Returns: 341 new XML node, suitable for call chaining 342 343 ***************************************************************************/ 344 345 private XmlNode xmlAddSkipped ( cstring name ) 346 { 347 if (this.xml_doc is null) 348 return null; 349 350 return this.xmlAddTestcase(name).element(null, "skipped"); 351 } 352 353 354 /************************************************************************** 355 356 Add a successful test node to the XML document 357 358 Params: 359 name = name of the test to add to the XML document 360 tv = time it took the test to run 361 362 Returns: 363 new XML node, suitable for call chaining 364 365 ***************************************************************************/ 366 367 private XmlNode xmlAddSuccess ( cstring name, timeval tv ) 368 { 369 if (this.xml_doc is null) 370 return null; 371 372 return this.xmlAddTestcase(name) 373 .attribute(null, "time", toXmlTime(tv).dup); 374 } 375 376 377 /************************************************************************** 378 379 Add a failed test node to the XML document 380 381 Params: 382 type = type of failure (either "failure" or "error") 383 name = name of the test to add to the XML document 384 tv = time it took the test to run 385 msg = reason why the test failed 386 387 Returns: 388 new XML node, suitable for call chaining 389 390 ***************************************************************************/ 391 392 private XmlNode xmlAddFailure (istring type = "failure") ( 393 cstring name, timeval tv, cstring msg ) 394 { 395 static assert (type == "failure" || type == "error"); 396 397 if (this.xml_doc is null) 398 return null; 399 400 // TODO: capture test output 401 return this.xmlAddSuccess(name, tv) 402 .element(null, type) 403 .attribute(null, "type", type) 404 .attribute(null, "message", msg.dup); 405 } 406 407 408 /************************************************************************** 409 410 Add a test node to the XML document 411 412 Params: 413 name = name of the test to add to the XML document 414 415 Returns: 416 new XML node, suitable for call chaining 417 418 ***************************************************************************/ 419 420 private XmlNode xmlAddTestcase ( cstring name ) 421 { 422 return this.xml_doc.elements 423 .element(null, "testcase") 424 .attribute(null, "classname", name) 425 .attribute(null, "name", "unittest"); 426 } 427 428 429 /************************************************************************** 430 431 Write the XML document to the file passed by command line arguments 432 433 Params: 434 tests = total amount of tests found 435 skipped = number of skipped tests 436 failures = number of failed tests 437 errors = number of errored tests 438 time = total time it took the tests to run 439 440 Returns: 441 true if the file was written successfully, false otherwise 442 443 ***************************************************************************/ 444 445 private bool writeXml ( size_t tests, size_t skipped, size_t failures, 446 size_t errors, timeval time) 447 { 448 if (this.xml_doc is null) 449 return true; 450 451 this.xml_doc.elements // root node: <testsuite> 452 .attribute(null, "tests", this.convert(tests).dup) 453 .attribute(null, "skipped", this.convert(skipped).dup) 454 .attribute(null, "failures", this.convert(failures).dup) 455 .attribute(null, "errors", this.convert(errors).dup) 456 .attribute(null, "time", this.toXmlTime(time).dup); 457 458 auto printer = new DocPrinter!(char); 459 try 460 { 461 auto output = new TextFileOutput(this.xml_file); 462 // At this point we don't care about errors anymore, is best effort 463 scope (failure) 464 { 465 // Make sure it ends with a null char, before passing it to 466 // the C function unlink() 467 this.xml_file ~= '\0'; 468 unlink(this.xml_file.ptr); 469 } 470 scope (exit) 471 { 472 // Workarround for the issue where buffered output is not flushed 473 // before close 474 output.flush(); 475 output.close(); 476 } 477 output.formatln("{}", printer.print(this.xml_doc)); 478 } 479 catch (Exception e) 480 { 481 Stderr.formatln("{}: error: writing XML file '{}': {} [{}:{}]", 482 this.prog, this.xml_file, e.message(), e.file, e.line); 483 return false; 484 } 485 486 return true; 487 } 488 489 490 /************************************************************************** 491 492 Convert a timeval to a string with a format suitable for the XML file 493 494 The format used is seconds, expressed as a floating point number, with 495 miliseconds resolution (i.e. 3 decimals precision). 496 497 Params: 498 tv = timeval to convert to string 499 500 Returns: 501 string with the XML compatible form of tv 502 503 ***************************************************************************/ 504 505 private cstring toXmlTime ( timeval tv ) 506 { 507 return this.convert(tv.tv_sec + tv.tv_usec / 1_000_000.0, "{:f3}"); 508 } 509 510 511 /************************************************************************** 512 513 Convert a timeval to a human readable string. 514 515 If it is in the order of hours, then "N.Nh" is used, if is in the order 516 of minutes, then "N.Nm" is used, and so on for seconds ("s" suffix), 517 milliseconds ("ms" suffix) and microseconds ("us" suffix). 518 519 Params: 520 tv = timeval to print 521 522 Returns: 523 string with the human readable form of tv. 524 525 ***************************************************************************/ 526 527 private cstring toHumanTime ( timeval tv ) 528 { 529 if (tv.tv_sec >= 60*60) 530 return this.convert(tv.tv_sec / 60.0 / 60.0, "{:f1}h"); 531 532 if (tv.tv_sec >= 60) 533 return this.convert(tv.tv_sec / 60.0, "{:f1}m"); 534 535 if (tv.tv_sec > 0) 536 return this.convert(tv.tv_sec + tv.tv_usec / 1_000_000.0, "{:f1}s"); 537 538 if (tv.tv_usec >= 1000) 539 return this.convert(tv.tv_usec / 1_000.0, "{:f1}ms"); 540 541 return this.convert(tv.tv_usec, "{}us"); 542 } 543 544 unittest 545 { 546 scope t = new UnitTestRunner; 547 timeval tv; 548 test!("==")(t.toHumanTime(tv), "0us"[]); 549 tv.tv_sec = 1; 550 test!("==")(t.toHumanTime(tv), "1.0s"[]); 551 tv.tv_sec = 1; 552 test!("==")(t.toHumanTime(tv), "1.0s"[]); 553 tv.tv_usec = 100_000; 554 test!("==")(t.toHumanTime(tv), "1.1s"[]); 555 tv.tv_usec = 561_235; 556 test!("==")(t.toHumanTime(tv), "1.6s"[]); 557 tv.tv_sec = 60; 558 test!("==")(t.toHumanTime(tv), "1.0m"[]); 559 tv.tv_sec = 61; 560 test!("==")(t.toHumanTime(tv), "1.0m"[]); 561 tv.tv_sec = 66; 562 test!("==")(t.toHumanTime(tv), "1.1m"[]); 563 tv.tv_sec = 60*60; 564 test!("==")(t.toHumanTime(tv), "1.0h"[]); 565 tv.tv_sec += 10; 566 test!("==")(t.toHumanTime(tv), "1.0h"[]); 567 tv.tv_sec += 6*60; 568 test!("==")(t.toHumanTime(tv), "1.1h"[]); 569 tv.tv_sec = 0; 570 test!("==")(t.toHumanTime(tv), "561.2ms"[]); 571 tv.tv_usec = 1_235; 572 test!("==")(t.toHumanTime(tv), "1.2ms"[]); 573 tv.tv_usec = 1_000; 574 test!("==")(t.toHumanTime(tv), "1.0ms"[]); 575 tv.tv_usec = 235; 576 test!("==")(t.toHumanTime(tv), "235us"[]); 577 } 578 579 580 /************************************************************************** 581 582 Convert an arbitrary value to string using the internal temporary buffer 583 584 Note: the return value can only be used temporarily, as it is stored in 585 the internal, reusable, buffer. 586 587 Params: 588 val = value to convert to string 589 fmt = Tango format string used to convert the value to string 590 591 Returns: 592 string with the value as specified by fmt 593 594 ***************************************************************************/ 595 596 private cstring convert ( T ) ( T val, cstring fmt = "{}" ) 597 { 598 this.buf.length = 0; 599 enableStomping(this.buf); 600 return sformat(this.buf, fmt, val); 601 } 602 603 604 /************************************************************************** 605 606 Possible test results. 607 608 ***************************************************************************/ 609 610 enum Result 611 { 612 Pass, 613 Fail, 614 Error, 615 } 616 617 /************************************************************************** 618 619 Test a single module, catching and reporting any errors. 620 621 Params: 622 m = module to be tested 623 tv = the time the test took to run will be written here 624 err = buffer where to write the error message if the test was 625 unsuccessful (is only written if the return value is != 626 Result.Pass 627 628 Returns: 629 the result of the test (passed, failure or error) 630 631 ***************************************************************************/ 632 633 private Result timedTest ( ModuleInfoPtr m, out timeval tv, ref mstring err ) 634 { 635 timeval start = this.now(); 636 scope (exit) tv = elapsedTime(start); 637 638 try 639 { 640 m.unitTest()(); 641 return Result.Pass; 642 } 643 catch (TestException e) 644 { 645 e.toString((d) { err ~= d; }); 646 return Result.Fail; 647 } 648 catch (SanityException e) 649 { 650 e.toString((d) { err ~= d; }); 651 } 652 catch (Exception e) 653 { 654 e.toString((d) { err ~= d; }); 655 } 656 657 return Result.Error; 658 } 659 660 661 /************************************************************************** 662 663 Gets the elapsed time between start and now 664 665 Returns: 666 a timeval with the elapsed time 667 668 ***************************************************************************/ 669 670 private static timeval elapsedTime ( timeval start ) 671 { 672 timeval elapsed; 673 timeval end = now(); 674 timersub(&end, &start, &elapsed); 675 676 return elapsed; 677 } 678 679 680 /************************************************************************** 681 682 Gets the current time with microseconds resolution 683 684 Returns: 685 a timeval representing the current date and time 686 687 ***************************************************************************/ 688 689 private static timeval now ( ) 690 { 691 timeval t; 692 int e = gettimeofday(&t, null); 693 verify(e == 0, "gettimeofday returned != 0"); 694 695 return t; 696 } 697 698 699 /************************************************************************** 700 701 Check if a module with name `name` should be tested. 702 703 Params: 704 name = Name of the module to check if it should be tested. 705 706 Returns: 707 true if it should be tested, false otherwise. 708 709 ***************************************************************************/ 710 711 bool shouldTest ( cstring name ) 712 { 713 // No packages specified, matches all 714 if (this.packages.length == 0) 715 return true; 716 717 foreach (pkg; this.packages) 718 { 719 // It matches as a module 720 if (name == pkg) 721 return true; 722 // If name is part of the package, it must start with "pkg." 723 if (name.length > pkg.length && 724 strncmp(pkg.ptr, name.ptr, pkg.length) == 0 && 725 name[pkg.length] == '.') 726 return true; 727 } 728 729 return false; 730 } 731 732 733 /************************************************************************** 734 735 Parse command line arguments filling the internal options and program 736 name. 737 738 This function also print help and error messages. 739 740 Params: 741 args = command line arguments as received by main() 742 743 Returns: 744 true if the arguments are OK, false otherwise. 745 746 ***************************************************************************/ 747 748 private bool parseArgs ( cstring[] args ) 749 { 750 // we don't care about freeing anything, is just a few bytes and the program 751 // will quite after we are done using these variables 752 auto prog = args[0].dup; 753 prog ~= '\0'; // Make sure is null-terminated 754 char* prog_c = basename(prog.ptr); 755 this.prog = prog_c[0..strlen(prog_c)]; 756 757 cstring[] unknown; 758 759 bool skip_next = false; 760 761 args = args[1..$]; 762 763 cstring getOptArg ( size_t i ) 764 { 765 if (args.length <= i+1) 766 { 767 this.printUsage(Stderr); 768 Stderr.formatln("\n{}: error: missing argument for {}", 769 this.prog, args[i]); 770 return null; 771 } 772 skip_next = true; 773 return args[i+1]; 774 } 775 776 foreach (i, arg; args) 777 { 778 if (skip_next) 779 { 780 skip_next = false; 781 continue; 782 } 783 784 switch (arg) 785 { 786 case "-h": 787 case "--help": 788 this.help = true; 789 this.printHelp(Stdout); 790 return true; 791 792 case "-vvv": 793 this.verbose++; 794 goto case; 795 case "-vv": 796 this.verbose++; 797 goto case; 798 case "-v": 799 case "--verbose": 800 this.verbose++; 801 break; 802 803 case "-s": 804 case "--summary": 805 this.summary = true; 806 break; 807 808 case "-k": 809 case "--keep-going": 810 this.keep_going = true; 811 break; 812 813 case "-p": 814 case "--package": 815 auto opt_arg = getOptArg(i); 816 if (opt_arg.length == 0) 817 return false; 818 // FIXME: This is just a compatibility-mode because at some 819 // point this worked implicitly as some sort of pattern 820 // matching ARG*, so to specify a strict package it was 821 // common to use "pkg.". Now we just remove the trailing 822 // "." if we find one, it's not a valid package name 823 // anyway. This was introduced in v3, so it should be 824 // safe to remove it in v4 or maybe v5 go make sure all 825 // dependencies were updated. 826 if (opt_arg[$-1] == '.') 827 opt_arg = opt_arg[0..$-1]; 828 this.packages ~= opt_arg; 829 break; 830 831 case "-x": 832 case "--xml-file": 833 this.xml_file = getOptArg(i); 834 if (this.xml_file is null) 835 return false; 836 if (this.xml_doc is null) 837 { 838 this.xml_doc = new XmlDoc; 839 this.xml_doc.header(); 840 this.xml_doc.tree.element(null, "testsuite") 841 .attribute(null, "name", "unittests"); 842 } 843 break; 844 845 default: 846 unknown ~= arg; 847 break; 848 } 849 } 850 851 if (unknown.length) 852 { 853 this.printUsage(Stderr); 854 Stderr.format("\n{}: error: Unknown arguments:", this.prog); 855 foreach (arg; unknown) 856 { 857 Stderr.format(" {}", arg); 858 } 859 Stderr.newline(); 860 return false; 861 } 862 863 // Remove any package duplicates 864 this.packages = uniq(this.packages); 865 866 return true; 867 } 868 869 870 /************************************************************************** 871 872 Print the program's usage string. 873 874 Params: 875 fp = File pointer where to print the usage. 876 877 ***************************************************************************/ 878 879 private void printUsage ( FormatOutput output ) 880 { 881 output.formatln("Usage: {} [-h] [-v] [-s] [-k] [-p PKG] [-x FILE]", 882 this.prog); 883 } 884 885 886 /************************************************************************** 887 888 Print the program's full help string. 889 890 Params: 891 fp = File pointer where to print the usage. 892 893 ***************************************************************************/ 894 895 private void printHelp ( FormatOutput output ) 896 { 897 this.printUsage(output); 898 output.format(` 899 optional arguments: 900 -h, --help print this message and exit 901 -v, --verbose print more information about unittest progress, can be 902 specified multiple times (even as -vvv, 3 is the maximum), 903 the first level only prints the executed tests, the second 904 level print the tests skipped because there are no unit 905 tests in the module or because it doesn't match the -p 906 patterns, and the third level print also tests skipped 907 because no -k is used and a test failed 908 -s, --summary print a summary with the passed, skipped and failed number 909 of tests 910 -k, --keep-going don't stop after the first module unittest failed 911 -p, --package PKG only run tests in package (or module) PKG (can be specified 912 multiple times) 913 -x, --xml-file FILE 914 write test results in FILE in a XML format that Jenkins 915 understands. 916 `); 917 } 918 } 919 920 921 922 /****************************************************************************** 923 924 Main function that run all the modules unittests using UnitTestRunner. 925 926 ******************************************************************************/ 927 928 int main(cstring[] args) 929 { 930 scope runner = new UnitTestRunner; 931 932 auto args_ok = runner.parseArgs(args); 933 934 if (runner.help) 935 return 0; 936 937 if (!args_ok) 938 return 2; 939 940 return runner.run(); 941 }