1 /****************************************************************************** 2 3 Flexible unittest runner 4 5 This module provides a more flexible unittest runner. 6 7 The goals for this test runner is to function as a standalone program, 8 instead of being run before another program's main(), as is common in D. 9 10 To achieve this, the main() function is provided by this module too. 11 12 To use it, just import this module and any other module you want to test, 13 for example: 14 15 --- 16 // Workaround: https://github.com/dlang/dub/issues/1761 17 module_ tester; 18 import ocean.core.UnitTestRunner; 19 import mymodule; 20 --- 21 22 That's it. Compile with: dmd -unittest tester.d mymodule.d 23 24 You can control the unittest execution, try ./tester -h for help on the 25 available options. 26 27 Tester status codes: 28 29 0 - All tests passed 30 2 - Wrong command line arguments 31 4 - One or more tests failed 32 8 - One or more tests had errors (unexpected problems) 33 16 - Error writing to XML file 34 35 Status codes can be aggregated via ORing, so for example, a status of 12 36 means both failures and errors were encountered, and status of 20 means 37 failures were encountered and there was an error writing the XML file. 38 39 Copyright: 40 Copyright (c) 2009-2016 dunnhumby Germany GmbH. 41 All rights reserved. 42 43 License: 44 Boost Software License Version 1.0. See LICENSE_BOOST.txt for details. 45 Alternatively, this file may be distributed under the terms of the Tango 46 3-Clause BSD License (see LICENSE_BSD.txt for details). 47 48 *******************************************************************************/ 49 50 module ocean.core.UnitTestRunner; 51 52 import ocean.core.array.Mutation: uniq; 53 import ocean.core.Test: TestException, test; 54 import ocean.core.Verify; 55 import ocean.io.Stdout: Stdout, Stderr; 56 import ocean.io.stream.Format: FormatOutput; 57 import ocean.io.stream.TextFile: TextFileOutput; 58 import ocean.text.xml.Document: Document; 59 import ocean.text.xml.DocPrinter: DocPrinter; 60 import ocean.text.convert.Formatter; 61 import ocean.transition; 62 63 import core.memory; 64 import core.runtime: Runtime; 65 import core.stdc.string: strdup, strlen, strncmp; 66 import core.sys.posix.unistd: unlink; 67 import core.sys.posix.sys.time: gettimeofday, timeval; 68 import core.sys.posix.libgen: basename; 69 import core.sys.posix.sys.time: timersub; 70 71 72 /****************************************************************************** 73 74 Handle all the details about unittest execution. 75 76 ******************************************************************************/ 77 78 private class UnitTestRunner 79 { 80 81 /************************************************************************** 82 83 Options parsed from the command-line 84 85 ***************************************************************************/ 86 87 private cstring prog; 88 private bool help = false; 89 private size_t verbose = 0; 90 private bool summary = false; 91 private bool keep_going = false; 92 private cstring[] packages = null; 93 private cstring xml_file; 94 95 96 /************************************************************************** 97 98 Aliases for easier access to symbols with long and intricate names 99 100 ***************************************************************************/ 101 102 private alias Document!(char) XmlDoc; 103 private alias XmlDoc.Node XmlNode; /// ditto 104 105 106 /************************************************************************** 107 108 Buffer used for text conversions 109 110 ***************************************************************************/ 111 112 private mstring buf; 113 114 115 /************************************************************************** 116 117 XML document used to produce the XML test results report 118 119 ***************************************************************************/ 120 121 private XmlDoc xml_doc; 122 123 124 /************************************************************************** 125 126 Static constructor replacing the default Tango unittest runner 127 128 ***************************************************************************/ 129 130 static this ( ) 131 { 132 Runtime.moduleUnitTester(&this.dummyUnitTestRunner); 133 } 134 135 136 /************************************************************************** 137 138 Dummy unittest runner. 139 140 This runner does nothing because we handle all the unittest execution 141 directly in the main() function, so we can parse the program's argument 142 before running the unittests. 143 144 Returns: 145 true to tell the runtime we want to run main() 146 147 ***************************************************************************/ 148 149 private static bool dummyUnitTestRunner() 150 { 151 return true; 152 } 153 154 155 /************************************************************************** 156 157 Run all the unittest registered by the runtime. 158 159 The parseArgs() function must be called before this method. 160 161 Returns: 162 exit status to pass to the operating system. 163 164 ***************************************************************************/ 165 166 private int run ( ) 167 { 168 verify(prog.length > 0); 169 170 timeval start_time = this.now(); 171 172 size_t passed = 0; 173 size_t failed = 0; 174 size_t errored = 0; 175 size_t skipped = 0; 176 size_t no_tests = 0; 177 size_t no_match = 0; 178 size_t gc_usage_before, gc_usage_after, mem_free; 179 bool collect_gc_usage = !!this.verbose; 180 181 if (this.verbose) 182 Stdout.formatln("{}: unit tests started", this.prog); 183 184 foreach ( m; ModuleInfo ) 185 { 186 if (!this.shouldTest(m.name)) 187 { 188 no_match++; 189 if (this.verbose > 1) 190 Stdout.formatln("{}: {}: skipped (not in " ~ 191 "modules to test)", this.prog, m.name); 192 continue; 193 } 194 195 if (m.unitTest is null) 196 { 197 no_tests++; 198 if (this.verbose > 1) 199 Stdout.formatln("{}: {}: skipped (no unittests)", 200 this.prog, m.name); 201 this.xmlAddSkipped(m.name); 202 continue; 203 } 204 205 if ((failed || errored) && !this.keep_going) 206 { 207 skipped++; 208 if (this.verbose > 2) 209 Stdout.formatln( 210 "{}: {}: skipped (one failed and no --keep-going)", 211 this.prog, m.name); 212 this.xmlAddSkipped(m.name); 213 continue; 214 } 215 216 if (this.verbose) 217 { 218 Stdout.format("{}: {}: testing ...", this.prog, m.name).flush(); 219 } 220 221 // we have a unittest, run it 222 timeval t; 223 // XXX: We can't use this.buf because it will be messed up when 224 // calling toHumanTime() and the different xmlAdd*() methods 225 static mstring e; 226 e.length = 0; 227 assumeSafeAppend(e); 228 scope (exit) 229 { 230 if (this.verbose) 231 Stdout.newline(); 232 if (e !is null) 233 Stdout.formatln("{}", e); 234 } 235 if (collect_gc_usage) 236 { 237 GC.collect(); 238 ocean.transition.gc_usage(gc_usage_before, mem_free); 239 } 240 switch (this.timedTest(m, t, e)) 241 { 242 case Result.Pass: 243 passed++; 244 if (this.verbose) 245 { 246 ocean.transition.gc_usage(gc_usage_after, mem_free); 247 Stdout.format(" PASS [{}, {} bytes ({} -> {})]", 248 this.toHumanTime(t), 249 cast(long)(gc_usage_after - gc_usage_before), 250 gc_usage_before, gc_usage_after); 251 } 252 this.xmlAddSuccess(m.name, t); 253 continue; 254 255 case Result.Fail: 256 failed++; 257 if (this.verbose) 258 Stdout.format(" FAIL [{}]", this.toHumanTime(t)); 259 this.xmlAddFailure(m.name, t, e); 260 break; 261 262 case Result.Error: 263 errored++; 264 if (this.verbose) 265 Stdout.format(" ERROR [{}]", this.toHumanTime(t)); 266 this.xmlAddFailure!("error")(m.name, t, e); 267 break; 268 269 default: 270 assert(false); 271 } 272 273 if (!this.keep_going) 274 continue; 275 276 if (this.verbose > 2) 277 Stdout.format(" (continuing, --keep-going used)"); 278 } 279 280 timeval total_time = elapsedTime(start_time); 281 282 if (this.summary) 283 { 284 Stdout.format("{}: {} modules passed, {} failed, " 285 ~ "{} with errors, {} without unittests", 286 this.prog, passed, failed, errored, no_tests); 287 if (!this.keep_going && failed) 288 Stdout.format(", {} skipped", skipped); 289 if (this.verbose > 1) 290 Stdout.format(", {} didn't match -p/--package", no_match); 291 Stdout.formatln(" [{}]", this.toHumanTime(total_time)); 292 } 293 294 bool xml_ok = this.writeXml( 295 passed + failed + errored + no_tests + skipped, 296 no_tests + skipped, failed, errored, total_time); 297 298 int ret = 0; 299 300 if (!xml_ok) 301 ret |= 16; 302 303 if (errored) 304 ret |= 8; 305 306 if (failed) 307 ret |= 4; 308 309 return ret; 310 } 311 312 313 /************************************************************************** 314 315 Add a skipped test node to the XML document 316 317 Params: 318 name = name of the test to add to the XML document 319 320 Returns: 321 new XML node, suitable for call chaining 322 323 ***************************************************************************/ 324 325 private XmlNode xmlAddSkipped ( cstring name ) 326 { 327 if (this.xml_doc is null) 328 return null; 329 330 return this.xmlAddTestcase(name).element(null, "skipped"); 331 } 332 333 334 /************************************************************************** 335 336 Add a successful test node to the XML document 337 338 Params: 339 name = name of the test to add to the XML document 340 tv = time it took the test to run 341 342 Returns: 343 new XML node, suitable for call chaining 344 345 ***************************************************************************/ 346 347 private XmlNode xmlAddSuccess ( cstring name, timeval tv ) 348 { 349 if (this.xml_doc is null) 350 return null; 351 352 return this.xmlAddTestcase(name) 353 .attribute(null, "time", toXmlTime(tv).dup); 354 } 355 356 357 /************************************************************************** 358 359 Add a failed test node to the XML document 360 361 Params: 362 type = type of failure (either "failure" or "error") 363 name = name of the test to add to the XML document 364 tv = time it took the test to run 365 msg = reason why the test failed 366 367 Returns: 368 new XML node, suitable for call chaining 369 370 ***************************************************************************/ 371 372 private XmlNode xmlAddFailure (istring type = "failure") ( 373 cstring name, timeval tv, cstring msg ) 374 { 375 static assert (type == "failure" || type == "error"); 376 377 if (this.xml_doc is null) 378 return null; 379 380 // TODO: capture test output 381 return this.xmlAddSuccess(name, tv) 382 .element(null, type) 383 .attribute(null, "type", type) 384 .attribute(null, "message", msg.dup); 385 } 386 387 388 /************************************************************************** 389 390 Add a test node to the XML document 391 392 Params: 393 name = name of the test to add to the XML document 394 395 Returns: 396 new XML node, suitable for call chaining 397 398 ***************************************************************************/ 399 400 private XmlNode xmlAddTestcase ( cstring name ) 401 { 402 return this.xml_doc.elements 403 .element(null, "testcase") 404 .attribute(null, "classname", name) 405 .attribute(null, "name", "unittest"); 406 } 407 408 409 /************************************************************************** 410 411 Write the XML document to the file passed by command line arguments 412 413 Params: 414 tests = total amount of tests found 415 skipped = number of skipped tests 416 failures = number of failed tests 417 errors = number of errored tests 418 time = total time it took the tests to run 419 420 Returns: 421 true if the file was written successfully, false otherwise 422 423 ***************************************************************************/ 424 425 private bool writeXml ( size_t tests, size_t skipped, size_t failures, 426 size_t errors, timeval time) 427 { 428 if (this.xml_doc is null) 429 return true; 430 431 this.xml_doc.elements // root node: <testsuite> 432 .attribute(null, "tests", this.convert(tests).dup) 433 .attribute(null, "skipped", this.convert(skipped).dup) 434 .attribute(null, "failures", this.convert(failures).dup) 435 .attribute(null, "errors", this.convert(errors).dup) 436 .attribute(null, "time", this.toXmlTime(time).dup); 437 438 auto printer = new DocPrinter!(char); 439 try 440 { 441 auto output = new TextFileOutput(this.xml_file); 442 // At this point we don't care about errors anymore, is best effort 443 scope (failure) 444 { 445 // Make sure it ends with a null char, before passing it to 446 // the C function unlink() 447 this.xml_file ~= '\0'; 448 unlink(this.xml_file.ptr); 449 } 450 scope (exit) 451 { 452 // Workarround for the issue where buffered output is not flushed 453 // before close 454 output.flush(); 455 output.close(); 456 } 457 output.formatln("{}", printer.print(this.xml_doc)); 458 } 459 catch (Exception e) 460 { 461 Stderr.formatln("{}: error: writing XML file '{}': {} [{}:{}]", 462 this.prog, this.xml_file, e.message(), e.file, e.line); 463 return false; 464 } 465 466 return true; 467 } 468 469 470 /************************************************************************** 471 472 Convert a timeval to a string with a format suitable for the XML file 473 474 The format used is seconds, expressed as a floating point number, with 475 miliseconds resolution (i.e. 3 decimals precision). 476 477 Params: 478 tv = timeval to convert to string 479 480 Returns: 481 string with the XML compatible form of tv 482 483 ***************************************************************************/ 484 485 private cstring toXmlTime ( timeval tv ) 486 { 487 return this.convert(tv.tv_sec + tv.tv_usec / 1_000_000.0, "{:f3}"); 488 } 489 490 491 /************************************************************************** 492 493 Convert a timeval to a human readable string. 494 495 If it is in the order of hours, then "N.Nh" is used, if is in the order 496 of minutes, then "N.Nm" is used, and so on for seconds ("s" suffix), 497 milliseconds ("ms" suffix) and microseconds ("us" suffix). 498 499 Params: 500 tv = timeval to print 501 502 Returns: 503 string with the human readable form of tv. 504 505 ***************************************************************************/ 506 507 private cstring toHumanTime ( timeval tv ) 508 { 509 if (tv.tv_sec >= 60*60) 510 return this.convert(tv.tv_sec / 60.0 / 60.0, "{:f1}h"); 511 512 if (tv.tv_sec >= 60) 513 return this.convert(tv.tv_sec / 60.0, "{:f1}m"); 514 515 if (tv.tv_sec > 0) 516 return this.convert(tv.tv_sec + tv.tv_usec / 1_000_000.0, "{:f1}s"); 517 518 if (tv.tv_usec >= 1000) 519 return this.convert(tv.tv_usec / 1_000.0, "{:f1}ms"); 520 521 return this.convert(tv.tv_usec, "{}us"); 522 } 523 524 unittest 525 { 526 scope t = new UnitTestRunner; 527 timeval tv; 528 test!("==")(t.toHumanTime(tv), "0us"[]); 529 tv.tv_sec = 1; 530 test!("==")(t.toHumanTime(tv), "1.0s"[]); 531 tv.tv_sec = 1; 532 test!("==")(t.toHumanTime(tv), "1.0s"[]); 533 tv.tv_usec = 100_000; 534 test!("==")(t.toHumanTime(tv), "1.1s"[]); 535 tv.tv_usec = 561_235; 536 test!("==")(t.toHumanTime(tv), "1.6s"[]); 537 tv.tv_sec = 60; 538 test!("==")(t.toHumanTime(tv), "1.0m"[]); 539 tv.tv_sec = 61; 540 test!("==")(t.toHumanTime(tv), "1.0m"[]); 541 tv.tv_sec = 66; 542 test!("==")(t.toHumanTime(tv), "1.1m"[]); 543 tv.tv_sec = 60*60; 544 test!("==")(t.toHumanTime(tv), "1.0h"[]); 545 tv.tv_sec += 10; 546 test!("==")(t.toHumanTime(tv), "1.0h"[]); 547 tv.tv_sec += 6*60; 548 test!("==")(t.toHumanTime(tv), "1.1h"[]); 549 tv.tv_sec = 0; 550 test!("==")(t.toHumanTime(tv), "561.2ms"[]); 551 tv.tv_usec = 1_235; 552 test!("==")(t.toHumanTime(tv), "1.2ms"[]); 553 tv.tv_usec = 1_000; 554 test!("==")(t.toHumanTime(tv), "1.0ms"[]); 555 tv.tv_usec = 235; 556 test!("==")(t.toHumanTime(tv), "235us"[]); 557 } 558 559 560 /************************************************************************** 561 562 Convert an arbitrary value to string using the internal temporary buffer 563 564 Note: the return value can only be used temporarily, as it is stored in 565 the internal, reusable, buffer. 566 567 Params: 568 val = value to convert to string 569 fmt = Tango format string used to convert the value to string 570 571 Returns: 572 string with the value as specified by fmt 573 574 ***************************************************************************/ 575 576 private cstring convert ( T ) ( T val, cstring fmt = "{}" ) 577 { 578 this.buf.length = 0; 579 assumeSafeAppend(this.buf); 580 return sformat(this.buf, fmt, val); 581 } 582 583 584 /************************************************************************** 585 586 Possible test results. 587 588 ***************************************************************************/ 589 590 enum Result 591 { 592 Pass, 593 Fail, 594 Error, 595 } 596 597 /************************************************************************** 598 599 Test a single module, catching and reporting any errors. 600 601 Params: 602 m = module to be tested 603 tv = the time the test took to run will be written here 604 err = buffer where to write the error message if the test was 605 unsuccessful (is only written if the return value is != 606 Result.Pass 607 608 Returns: 609 the result of the test (passed, failure or error) 610 611 ***************************************************************************/ 612 613 private Result timedTest ( ModuleInfo* m, out timeval tv, ref mstring err ) 614 { 615 timeval start = this.now(); 616 scope (exit) tv = elapsedTime(start); 617 618 try 619 { 620 m.unitTest()(); 621 return Result.Pass; 622 } 623 catch (TestException e) 624 { 625 e.toString((d) { err ~= d; }); 626 return Result.Fail; 627 } 628 catch (SanityException e) 629 { 630 e.toString((d) { err ~= d; }); 631 } 632 catch (Exception e) 633 { 634 e.toString((d) { err ~= d; }); 635 } 636 637 return Result.Error; 638 } 639 640 641 /************************************************************************** 642 643 Gets the elapsed time between start and now 644 645 Returns: 646 a timeval with the elapsed time 647 648 ***************************************************************************/ 649 650 private static timeval elapsedTime ( timeval start ) 651 { 652 timeval elapsed; 653 timeval end = now(); 654 timersub(&end, &start, &elapsed); 655 656 return elapsed; 657 } 658 659 660 /************************************************************************** 661 662 Gets the current time with microseconds resolution 663 664 Returns: 665 a timeval representing the current date and time 666 667 ***************************************************************************/ 668 669 private static timeval now ( ) 670 { 671 timeval t; 672 int e = gettimeofday(&t, null); 673 verify(e == 0, "gettimeofday returned != 0"); 674 675 return t; 676 } 677 678 679 /************************************************************************** 680 681 Check if a module with name `name` should be tested. 682 683 Params: 684 name = Name of the module to check if it should be tested. 685 686 Returns: 687 true if it should be tested, false otherwise. 688 689 ***************************************************************************/ 690 691 bool shouldTest ( cstring name ) 692 { 693 // No packages specified, matches all 694 if (this.packages.length == 0) 695 return true; 696 697 foreach (pkg; this.packages) 698 { 699 // It matches as a module 700 if (name == pkg) 701 return true; 702 // If name is part of the package, it must start with "pkg." 703 if (name.length > pkg.length && 704 strncmp(pkg.ptr, name.ptr, pkg.length) == 0 && 705 name[pkg.length] == '.') 706 return true; 707 } 708 709 return false; 710 } 711 712 713 /************************************************************************** 714 715 Parse command line arguments filling the internal options and program 716 name. 717 718 This function also print help and error messages. 719 720 Params: 721 args = command line arguments as received by main() 722 723 Returns: 724 true if the arguments are OK, false otherwise. 725 726 ***************************************************************************/ 727 728 private bool parseArgs ( cstring[] args ) 729 { 730 // we don't care about freeing anything, is just a few bytes and the program 731 // will quite after we are done using these variables 732 auto prog = args[0].dup; 733 prog ~= '\0'; // Make sure is null-terminated 734 char* prog_c = basename(prog.ptr); 735 this.prog = prog_c[0..strlen(prog_c)]; 736 737 cstring[] unknown; 738 739 bool skip_next = false; 740 741 args = args[1..$]; 742 743 cstring getOptArg ( size_t i ) 744 { 745 if (args.length <= i+1) 746 { 747 this.printUsage(Stderr); 748 Stderr.formatln("\n{}: error: missing argument for {}", 749 this.prog, args[i]); 750 return null; 751 } 752 skip_next = true; 753 return args[i+1]; 754 } 755 756 foreach (i, arg; args) 757 { 758 if (skip_next) 759 { 760 skip_next = false; 761 continue; 762 } 763 764 switch (arg) 765 { 766 case "-h": 767 case "--help": 768 this.help = true; 769 this.printHelp(Stdout); 770 return true; 771 772 case "-vvv": 773 this.verbose++; 774 goto case; 775 case "-vv": 776 this.verbose++; 777 goto case; 778 case "-v": 779 case "--verbose": 780 this.verbose++; 781 break; 782 783 case "-s": 784 case "--summary": 785 this.summary = true; 786 break; 787 788 case "-k": 789 case "--keep-going": 790 this.keep_going = true; 791 break; 792 793 case "-p": 794 case "--package": 795 auto opt_arg = getOptArg(i); 796 if (opt_arg.length == 0) 797 return false; 798 // FIXME: This is just a compatibility-mode because at some 799 // point this worked implicitly as some sort of pattern 800 // matching ARG*, so to specify a strict package it was 801 // common to use "pkg.". Now we just remove the trailing 802 // "." if we find one, it's not a valid package name 803 // anyway. This was introduced in v3, so it should be 804 // safe to remove it in v4 or maybe v5 go make sure all 805 // dependencies were updated. 806 if (opt_arg[$-1] == '.') 807 opt_arg = opt_arg[0..$-1]; 808 this.packages ~= opt_arg; 809 break; 810 811 case "-x": 812 case "--xml-file": 813 this.xml_file = getOptArg(i); 814 if (this.xml_file is null) 815 return false; 816 if (this.xml_doc is null) 817 { 818 this.xml_doc = new XmlDoc; 819 this.xml_doc.header(); 820 this.xml_doc.tree.element(null, "testsuite") 821 .attribute(null, "name", "unittests"); 822 } 823 break; 824 825 default: 826 unknown ~= arg; 827 break; 828 } 829 } 830 831 if (unknown.length) 832 { 833 this.printUsage(Stderr); 834 Stderr.format("\n{}: error: Unknown arguments:", this.prog); 835 foreach (arg; unknown) 836 { 837 Stderr.format(" {}", arg); 838 } 839 Stderr.newline(); 840 return false; 841 } 842 843 // Remove any package duplicates 844 this.packages = uniq(this.packages); 845 846 return true; 847 } 848 849 850 /************************************************************************** 851 852 Print the program's usage string. 853 854 Params: 855 fp = File pointer where to print the usage. 856 857 ***************************************************************************/ 858 859 private void printUsage ( FormatOutput output ) 860 { 861 output.formatln("Usage: {} [-h] [-v] [-s] [-k] [-p PKG] [-x FILE]", 862 this.prog); 863 } 864 865 866 /************************************************************************** 867 868 Print the program's full help string. 869 870 Params: 871 fp = File pointer where to print the usage. 872 873 ***************************************************************************/ 874 875 private void printHelp ( FormatOutput output ) 876 { 877 this.printUsage(output); 878 output.format(` 879 optional arguments: 880 -h, --help print this message and exit 881 -v, --verbose print more information about unittest progress, can be 882 specified multiple times (even as -vvv, 3 is the maximum), 883 the first level only prints the executed tests, the second 884 level print the tests skipped because there are no unit 885 tests in the module or because it doesn't match the -p 886 patterns, and the third level print also tests skipped 887 because no -k is used and a test failed 888 -s, --summary print a summary with the passed, skipped and failed number 889 of tests 890 -k, --keep-going don't stop after the first module unittest failed 891 -p, --package PKG only run tests in package (or module) PKG (can be specified 892 multiple times) 893 -x, --xml-file FILE 894 write test results in FILE in a XML format that Jenkins 895 understands. 896 `); 897 } 898 } 899 900 901 902 /****************************************************************************** 903 904 Main function that run all the modules unittests using UnitTestRunner. 905 906 ******************************************************************************/ 907 908 int main(cstring[] args) 909 { 910 scope runner = new UnitTestRunner; 911 912 auto args_ok = runner.parseArgs(args); 913 914 if (runner.help) 915 return 0; 916 917 if (!args_ok) 918 return 2; 919 920 return runner.run(); 921 }