1 /******************************************************************************
2 3 Flexible unittest runner
4 5 This module provides a more flexible unittest runner.
6 7 The goals for this test runner is to function as a standalone program,
8 instead of being run before another program's main(), as is common in D.
9 10 To achieve this, the main() function is provided by this module too.
11 12 To use it, just import this module and any other module you want to test,
13 for example:
14 15 ---
16 // Workaround: https://github.com/dlang/dub/issues/1761
17 module_ tester;
18 import ocean.core.UnitTestRunner;
19 import mymodule;
20 ---
21 22 That's it. Compile with: dmd -unittest tester.d mymodule.d
23 24 You can control the unittest execution, try ./tester -h for help on the
25 available options.
26 27 Tester status codes:
28 29 0 - All tests passed
30 2 - Wrong command line arguments
31 4 - One or more tests failed
32 8 - One or more tests had errors (unexpected problems)
33 16 - Error writing to XML file
34 35 Status codes can be aggregated via ORing, so for example, a status of 12
36 means both failures and errors were encountered, and status of 20 means
37 failures were encountered and there was an error writing the XML file.
38 39 Copyright:
40 Copyright (c) 2009-2016 dunnhumby Germany GmbH.
41 All rights reserved.
42 43 License:
44 Boost Software License Version 1.0. See LICENSE_BOOST.txt for details.
45 Alternatively, this file may be distributed under the terms of the Tango
46 3-Clause BSD License (see LICENSE_BSD.txt for details).
47 48 *******************************************************************************/49 50 moduleocean.core.UnitTestRunner;
51 52 importocean.core.array.Mutation: uniq;
53 importocean.core.Test: TestException, test;
54 importocean.core.Verify;
55 importocean.io.Stdout: Stdout, Stderr;
56 importocean.io.stream.Format: FormatOutput;
57 importocean.io.stream.TextFile: TextFileOutput;
58 importocean.text.xml.Document: Document;
59 importocean.text.xml.DocPrinter: DocPrinter;
60 importocean.text.convert.Formatter;
61 importocean.transition;
62 63 importcore.memory;
64 importcore.runtime: Runtime;
65 importcore.stdc.string: strdup, strlen, strncmp;
66 importcore.sys.posix.unistd: unlink;
67 importcore.sys.posix.sys.time: gettimeofday, timeval;
68 importcore.sys.posix.libgen: basename;
69 importcore.sys.posix.sys.time: timersub;
70 71 72 /******************************************************************************
73 74 Handle all the details about unittest execution.
75 76 ******************************************************************************/77 78 privateclassUnitTestRunner79 {
80 81 /**************************************************************************
82 83 Options parsed from the command-line
84 85 ***************************************************************************/86 87 privatecstringprog;
88 privateboolhelp = false;
89 privatesize_tverbose = 0;
90 privateboolsummary = false;
91 privateboolkeep_going = false;
92 privatecstring[] packages = null;
93 privatecstringxml_file;
94 95 96 /**************************************************************************
97 98 Aliases for easier access to symbols with long and intricate names
99 100 ***************************************************************************/101 102 privatealiasDocument!(char) XmlDoc;
103 privatealiasXmlDoc.NodeXmlNode; /// ditto104 105 106 /**************************************************************************
107 108 Buffer used for text conversions
109 110 ***************************************************************************/111 112 privatemstringbuf;
113 114 115 /**************************************************************************
116 117 XML document used to produce the XML test results report
118 119 ***************************************************************************/120 121 privateXmlDocxml_doc;
122 123 124 /**************************************************************************
125 126 Static constructor replacing the default Tango unittest runner
127 128 ***************************************************************************/129 130 staticthis ( )
131 {
132 Runtime.moduleUnitTester(&this.dummyUnitTestRunner);
133 }
134 135 136 /**************************************************************************
137 138 Dummy unittest runner.
139 140 This runner does nothing because we handle all the unittest execution
141 directly in the main() function, so we can parse the program's argument
142 before running the unittests.
143 144 Returns:
145 true to tell the runtime we want to run main()
146 147 ***************************************************************************/148 149 privatestaticbooldummyUnitTestRunner()
150 {
151 returntrue;
152 }
153 154 155 /**************************************************************************
156 157 Run all the unittest registered by the runtime.
158 159 The parseArgs() function must be called before this method.
160 161 Returns:
162 exit status to pass to the operating system.
163 164 ***************************************************************************/165 166 privateintrun ( )
167 {
168 verify(prog.length > 0);
169 170 timevalstart_time = this.now();
171 172 size_tpassed = 0;
173 size_tfailed = 0;
174 size_terrored = 0;
175 size_tskipped = 0;
176 size_tno_tests = 0;
177 size_tno_match = 0;
178 size_tgc_usage_before, gc_usage_after, mem_free;
179 boolcollect_gc_usage = !!this.verbose;
180 181 if (this.verbose)
182 Stdout.formatln("{}: unit tests started", this.prog);
183 184 foreach ( m; ModuleInfo )
185 {
186 if (!this.shouldTest(m.name))
187 {
188 no_match++;
189 if (this.verbose > 1)
190 Stdout.formatln("{}: {}: skipped (not in " ~
191 "modules to test)", this.prog, m.name);
192 continue;
193 }
194 195 if (m.unitTestisnull)
196 {
197 no_tests++;
198 if (this.verbose > 1)
199 Stdout.formatln("{}: {}: skipped (no unittests)",
200 this.prog, m.name);
201 this.xmlAddSkipped(m.name);
202 continue;
203 }
204 205 if ((failed || errored) && !this.keep_going)
206 {
207 skipped++;
208 if (this.verbose > 2)
209 Stdout.formatln(
210 "{}: {}: skipped (one failed and no --keep-going)",
211 this.prog, m.name);
212 this.xmlAddSkipped(m.name);
213 continue;
214 }
215 216 if (this.verbose)
217 {
218 Stdout.format("{}: {}: testing ...", this.prog, m.name).flush();
219 }
220 221 // we have a unittest, run it222 timevalt;
223 // XXX: We can't use this.buf because it will be messed up when224 // calling toHumanTime() and the different xmlAdd*() methods225 staticmstringe;
226 e.length = 0;
227 assumeSafeAppend(e);
228 scope (exit)
229 {
230 if (this.verbose)
231 Stdout.newline();
232 if (e !isnull)
233 Stdout.formatln("{}", e);
234 }
235 if (collect_gc_usage)
236 {
237 GC.collect();
238 ocean.transition.gc_usage(gc_usage_before, mem_free);
239 }
240 switch (this.timedTest(m, t, e))
241 {
242 caseResult.Pass:
243 passed++;
244 if (this.verbose)
245 {
246 ocean.transition.gc_usage(gc_usage_after, mem_free);
247 Stdout.format(" PASS [{}, {} bytes ({} -> {})]",
248 this.toHumanTime(t),
249 cast(long)(gc_usage_after - gc_usage_before),
250 gc_usage_before, gc_usage_after);
251 }
252 this.xmlAddSuccess(m.name, t);
253 continue;
254 255 caseResult.Fail:
256 failed++;
257 if (this.verbose)
258 Stdout.format(" FAIL [{}]", this.toHumanTime(t));
259 this.xmlAddFailure(m.name, t, e);
260 break;
261 262 caseResult.Error:
263 errored++;
264 if (this.verbose)
265 Stdout.format(" ERROR [{}]", this.toHumanTime(t));
266 this.xmlAddFailure!("error")(m.name, t, e);
267 break;
268 269 default:
270 assert(false);
271 }
272 273 if (!this.keep_going)
274 continue;
275 276 if (this.verbose > 2)
277 Stdout.format(" (continuing, --keep-going used)");
278 }
279 280 timevaltotal_time = elapsedTime(start_time);
281 282 if (this.summary)
283 {
284 Stdout.format("{}: {} modules passed, {} failed, "285 ~ "{} with errors, {} without unittests",
286 this.prog, passed, failed, errored, no_tests);
287 if (!this.keep_going && failed)
288 Stdout.format(", {} skipped", skipped);
289 if (this.verbose > 1)
290 Stdout.format(", {} didn't match -p/--package", no_match);
291 Stdout.formatln(" [{}]", this.toHumanTime(total_time));
292 }
293 294 boolxml_ok = this.writeXml(
295 passed + failed + errored + no_tests + skipped,
296 no_tests + skipped, failed, errored, total_time);
297 298 intret = 0;
299 300 if (!xml_ok)
301 ret |= 16;
302 303 if (errored)
304 ret |= 8;
305 306 if (failed)
307 ret |= 4;
308 309 returnret;
310 }
311 312 313 /**************************************************************************
314 315 Add a skipped test node to the XML document
316 317 Params:
318 name = name of the test to add to the XML document
319 320 Returns:
321 new XML node, suitable for call chaining
322 323 ***************************************************************************/324 325 privateXmlNodexmlAddSkipped ( cstringname )
326 {
327 if (this.xml_docisnull)
328 returnnull;
329 330 returnthis.xmlAddTestcase(name).element(null, "skipped");
331 }
332 333 334 /**************************************************************************
335 336 Add a successful test node to the XML document
337 338 Params:
339 name = name of the test to add to the XML document
340 tv = time it took the test to run
341 342 Returns:
343 new XML node, suitable for call chaining
344 345 ***************************************************************************/346 347 privateXmlNodexmlAddSuccess ( cstringname, timevaltv )
348 {
349 if (this.xml_docisnull)
350 returnnull;
351 352 returnthis.xmlAddTestcase(name)
353 .attribute(null, "time", toXmlTime(tv).dup);
354 }
355 356 357 /**************************************************************************
358 359 Add a failed test node to the XML document
360 361 Params:
362 type = type of failure (either "failure" or "error")
363 name = name of the test to add to the XML document
364 tv = time it took the test to run
365 msg = reason why the test failed
366 367 Returns:
368 new XML node, suitable for call chaining
369 370 ***************************************************************************/371 372 privateXmlNodexmlAddFailure (istringtype = "failure") (
373 cstringname, timevaltv, cstringmsg )
374 {
375 staticassert (type == "failure" || type == "error");
376 377 if (this.xml_docisnull)
378 returnnull;
379 380 // TODO: capture test output381 returnthis.xmlAddSuccess(name, tv)
382 .element(null, type)
383 .attribute(null, "type", type)
384 .attribute(null, "message", msg.dup);
385 }
386 387 388 /**************************************************************************
389 390 Add a test node to the XML document
391 392 Params:
393 name = name of the test to add to the XML document
394 395 Returns:
396 new XML node, suitable for call chaining
397 398 ***************************************************************************/399 400 privateXmlNodexmlAddTestcase ( cstringname )
401 {
402 returnthis.xml_doc.elements403 .element(null, "testcase")
404 .attribute(null, "classname", name)
405 .attribute(null, "name", "unittest");
406 }
407 408 409 /**************************************************************************
410 411 Write the XML document to the file passed by command line arguments
412 413 Params:
414 tests = total amount of tests found
415 skipped = number of skipped tests
416 failures = number of failed tests
417 errors = number of errored tests
418 time = total time it took the tests to run
419 420 Returns:
421 true if the file was written successfully, false otherwise
422 423 ***************************************************************************/424 425 privateboolwriteXml ( size_ttests, size_tskipped, size_tfailures,
426 size_terrors, timevaltime)
427 {
428 if (this.xml_docisnull)
429 returntrue;
430 431 this.xml_doc.elements// root node: <testsuite>432 .attribute(null, "tests", this.convert(tests).dup)
433 .attribute(null, "skipped", this.convert(skipped).dup)
434 .attribute(null, "failures", this.convert(failures).dup)
435 .attribute(null, "errors", this.convert(errors).dup)
436 .attribute(null, "time", this.toXmlTime(time).dup);
437 438 autoprinter = newDocPrinter!(char);
439 try440 {
441 autooutput = newTextFileOutput(this.xml_file);
442 // At this point we don't care about errors anymore, is best effort443 scope (failure)
444 {
445 // Make sure it ends with a null char, before passing it to446 // the C function unlink()447 this.xml_file ~= '\0';
448 unlink(this.xml_file.ptr);
449 }
450 scope (exit)
451 {
452 // Workarround for the issue where buffered output is not flushed453 // before close454 output.flush();
455 output.close();
456 }
457 output.formatln("{}", printer.print(this.xml_doc));
458 }
459 catch (Exceptione)
460 {
461 Stderr.formatln("{}: error: writing XML file '{}': {} [{}:{}]",
462 this.prog, this.xml_file, e.message(), e.file, e.line);
463 returnfalse;
464 }
465 466 returntrue;
467 }
468 469 470 /**************************************************************************
471 472 Convert a timeval to a string with a format suitable for the XML file
473 474 The format used is seconds, expressed as a floating point number, with
475 miliseconds resolution (i.e. 3 decimals precision).
476 477 Params:
478 tv = timeval to convert to string
479 480 Returns:
481 string with the XML compatible form of tv
482 483 ***************************************************************************/484 485 privatecstringtoXmlTime ( timevaltv )
486 {
487 returnthis.convert(tv.tv_sec + tv.tv_usec / 1_000_000.0, "{:f3}");
488 }
489 490 491 /**************************************************************************
492 493 Convert a timeval to a human readable string.
494 495 If it is in the order of hours, then "N.Nh" is used, if is in the order
496 of minutes, then "N.Nm" is used, and so on for seconds ("s" suffix),
497 milliseconds ("ms" suffix) and microseconds ("us" suffix).
498 499 Params:
500 tv = timeval to print
501 502 Returns:
503 string with the human readable form of tv.
504 505 ***************************************************************************/506 507 privatecstringtoHumanTime ( timevaltv )
508 {
509 if (tv.tv_sec >= 60*60)
510 returnthis.convert(tv.tv_sec / 60.0 / 60.0, "{:f1}h");
511 512 if (tv.tv_sec >= 60)
513 returnthis.convert(tv.tv_sec / 60.0, "{:f1}m");
514 515 if (tv.tv_sec > 0)
516 returnthis.convert(tv.tv_sec + tv.tv_usec / 1_000_000.0, "{:f1}s");
517 518 if (tv.tv_usec >= 1000)
519 returnthis.convert(tv.tv_usec / 1_000.0, "{:f1}ms");
520 521 returnthis.convert(tv.tv_usec, "{}us");
522 }
523 524 unittest525 {
526 scopet = newUnitTestRunner;
527 timevaltv;
528 test!("==")(t.toHumanTime(tv), "0us"[]);
529 tv.tv_sec = 1;
530 test!("==")(t.toHumanTime(tv), "1.0s"[]);
531 tv.tv_sec = 1;
532 test!("==")(t.toHumanTime(tv), "1.0s"[]);
533 tv.tv_usec = 100_000;
534 test!("==")(t.toHumanTime(tv), "1.1s"[]);
535 tv.tv_usec = 561_235;
536 test!("==")(t.toHumanTime(tv), "1.6s"[]);
537 tv.tv_sec = 60;
538 test!("==")(t.toHumanTime(tv), "1.0m"[]);
539 tv.tv_sec = 61;
540 test!("==")(t.toHumanTime(tv), "1.0m"[]);
541 tv.tv_sec = 66;
542 test!("==")(t.toHumanTime(tv), "1.1m"[]);
543 tv.tv_sec = 60*60;
544 test!("==")(t.toHumanTime(tv), "1.0h"[]);
545 tv.tv_sec += 10;
546 test!("==")(t.toHumanTime(tv), "1.0h"[]);
547 tv.tv_sec += 6*60;
548 test!("==")(t.toHumanTime(tv), "1.1h"[]);
549 tv.tv_sec = 0;
550 test!("==")(t.toHumanTime(tv), "561.2ms"[]);
551 tv.tv_usec = 1_235;
552 test!("==")(t.toHumanTime(tv), "1.2ms"[]);
553 tv.tv_usec = 1_000;
554 test!("==")(t.toHumanTime(tv), "1.0ms"[]);
555 tv.tv_usec = 235;
556 test!("==")(t.toHumanTime(tv), "235us"[]);
557 }
558 559 560 /**************************************************************************
561 562 Convert an arbitrary value to string using the internal temporary buffer
563 564 Note: the return value can only be used temporarily, as it is stored in
565 the internal, reusable, buffer.
566 567 Params:
568 val = value to convert to string
569 fmt = Tango format string used to convert the value to string
570 571 Returns:
572 string with the value as specified by fmt
573 574 ***************************************************************************/575 576 privatecstringconvert ( T ) ( Tval, cstringfmt = "{}" )
577 {
578 this.buf.length = 0;
579 assumeSafeAppend(this.buf);
580 returnsformat(this.buf, fmt, val);
581 }
582 583 584 /**************************************************************************
585 586 Possible test results.
587 588 ***************************************************************************/589 590 enumResult591 {
592 Pass,
593 Fail,
594 Error,
595 }
596 597 /**************************************************************************
598 599 Test a single module, catching and reporting any errors.
600 601 Params:
602 m = module to be tested
603 tv = the time the test took to run will be written here
604 err = buffer where to write the error message if the test was
605 unsuccessful (is only written if the return value is !=
606 Result.Pass
607 608 Returns:
609 the result of the test (passed, failure or error)
610 611 ***************************************************************************/612 613 privateResulttimedTest ( ModuleInfo* m, outtimevaltv, refmstringerr )
614 {
615 timevalstart = this.now();
616 scope (exit) tv = elapsedTime(start);
617 618 try619 {
620 m.unitTest()();
621 returnResult.Pass;
622 }
623 catch (TestExceptione)
624 {
625 e.toString((d) { err ~= d; });
626 returnResult.Fail;
627 }
628 catch (SanityExceptione)
629 {
630 e.toString((d) { err ~= d; });
631 }
632 catch (Exceptione)
633 {
634 e.toString((d) { err ~= d; });
635 }
636 637 returnResult.Error;
638 }
639 640 641 /**************************************************************************
642 643 Gets the elapsed time between start and now
644 645 Returns:
646 a timeval with the elapsed time
647 648 ***************************************************************************/649 650 privatestatictimevalelapsedTime ( timevalstart )
651 {
652 timevalelapsed;
653 timevalend = now();
654 timersub(&end, &start, &elapsed);
655 656 returnelapsed;
657 }
658 659 660 /**************************************************************************
661 662 Gets the current time with microseconds resolution
663 664 Returns:
665 a timeval representing the current date and time
666 667 ***************************************************************************/668 669 privatestatictimevalnow ( )
670 {
671 timevalt;
672 inte = gettimeofday(&t, null);
673 verify(e == 0, "gettimeofday returned != 0");
674 675 returnt;
676 }
677 678 679 /**************************************************************************
680 681 Check if a module with name `name` should be tested.
682 683 Params:
684 name = Name of the module to check if it should be tested.
685 686 Returns:
687 true if it should be tested, false otherwise.
688 689 ***************************************************************************/690 691 boolshouldTest ( cstringname )
692 {
693 // No packages specified, matches all694 if (this.packages.length == 0)
695 returntrue;
696 697 foreach (pkg; this.packages)
698 {
699 // It matches as a module700 if (name == pkg)
701 returntrue;
702 // If name is part of the package, it must start with "pkg."703 if (name.length > pkg.length &&
704 strncmp(pkg.ptr, name.ptr, pkg.length) == 0 &&
705 name[pkg.length] == '.')
706 returntrue;
707 }
708 709 returnfalse;
710 }
711 712 713 /**************************************************************************
714 715 Parse command line arguments filling the internal options and program
716 name.
717 718 This function also print help and error messages.
719 720 Params:
721 args = command line arguments as received by main()
722 723 Returns:
724 true if the arguments are OK, false otherwise.
725 726 ***************************************************************************/727 728 privateboolparseArgs ( cstring[] args )
729 {
730 // we don't care about freeing anything, is just a few bytes and the program731 // will quite after we are done using these variables732 autoprog = args[0].dup;
733 prog ~= '\0'; // Make sure is null-terminated734 char* prog_c = basename(prog.ptr);
735 this.prog = prog_c[0..strlen(prog_c)];
736 737 cstring[] unknown;
738 739 boolskip_next = false;
740 741 args = args[1..$];
742 743 cstringgetOptArg ( size_ti )
744 {
745 if (args.length <= i+1)
746 {
747 this.printUsage(Stderr);
748 Stderr.formatln("\n{}: error: missing argument for {}",
749 this.prog, args[i]);
750 returnnull;
751 }
752 skip_next = true;
753 returnargs[i+1];
754 }
755 756 foreach (i, arg; args)
757 {
758 if (skip_next)
759 {
760 skip_next = false;
761 continue;
762 }
763 764 switch (arg)
765 {
766 case"-h":
767 case"--help":
768 this.help = true;
769 this.printHelp(Stdout);
770 returntrue;
771 772 case"-vvv":
773 this.verbose++;
774 gotocase;
775 case"-vv":
776 this.verbose++;
777 gotocase;
778 case"-v":
779 case"--verbose":
780 this.verbose++;
781 break;
782 783 case"-s":
784 case"--summary":
785 this.summary = true;
786 break;
787 788 case"-k":
789 case"--keep-going":
790 this.keep_going = true;
791 break;
792 793 case"-p":
794 case"--package":
795 autoopt_arg = getOptArg(i);
796 if (opt_arg.length == 0)
797 returnfalse;
798 // FIXME: This is just a compatibility-mode because at some799 // point this worked implicitly as some sort of pattern800 // matching ARG*, so to specify a strict package it was801 // common to use "pkg.". Now we just remove the trailing802 // "." if we find one, it's not a valid package name803 // anyway. This was introduced in v3, so it should be804 // safe to remove it in v4 or maybe v5 go make sure all805 // dependencies were updated.806 if (opt_arg[$-1] == '.')
807 opt_arg = opt_arg[0..$-1];
808 this.packages ~= opt_arg;
809 break;
810 811 case"-x":
812 case"--xml-file":
813 this.xml_file = getOptArg(i);
814 if (this.xml_fileisnull)
815 returnfalse;
816 if (this.xml_docisnull)
817 {
818 this.xml_doc = newXmlDoc;
819 this.xml_doc.header();
820 this.xml_doc.tree.element(null, "testsuite")
821 .attribute(null, "name", "unittests");
822 }
823 break;
824 825 default:
826 unknown ~= arg;
827 break;
828 }
829 }
830 831 if (unknown.length)
832 {
833 this.printUsage(Stderr);
834 Stderr.format("\n{}: error: Unknown arguments:", this.prog);
835 foreach (arg; unknown)
836 {
837 Stderr.format(" {}", arg);
838 }
839 Stderr.newline();
840 returnfalse;
841 }
842 843 // Remove any package duplicates844 this.packages = uniq(this.packages);
845 846 returntrue;
847 }
848 849 850 /**************************************************************************
851 852 Print the program's usage string.
853 854 Params:
855 fp = File pointer where to print the usage.
856 857 ***************************************************************************/858 859 privatevoidprintUsage ( FormatOutputoutput )
860 {
861 output.formatln("Usage: {} [-h] [-v] [-s] [-k] [-p PKG] [-x FILE]",
862 this.prog);
863 }
864 865 866 /**************************************************************************
867 868 Print the program's full help string.
869 870 Params:
871 fp = File pointer where to print the usage.
872 873 ***************************************************************************/874 875 privatevoidprintHelp ( FormatOutputoutput )
876 {
877 this.printUsage(output);
878 output.format(`
879 optional arguments:
880 -h, --help print this message and exit
881 -v, --verbose print more information about unittest progress, can be
882 specified multiple times (even as -vvv, 3 is the maximum),
883 the first level only prints the executed tests, the second
884 level print the tests skipped because there are no unit
885 tests in the module or because it doesn't match the -p
886 patterns, and the third level print also tests skipped
887 because no -k is used and a test failed
888 -s, --summary print a summary with the passed, skipped and failed number
889 of tests
890 -k, --keep-going don't stop after the first module unittest failed
891 -p, --package PKG only run tests in package (or module) PKG (can be specified
892 multiple times)
893 -x, --xml-file FILE
894 write test results in FILE in a XML format that Jenkins
895 understands.
896 `);
897 }
898 }
899 900 901 902 /******************************************************************************
903 904 Main function that run all the modules unittests using UnitTestRunner.
905 906 ******************************************************************************/907 908 intmain(cstring[] args)
909 {
910 scoperunner = newUnitTestRunner;
911 912 autoargs_ok = runner.parseArgs(args);
913 914 if (runner.help)
915 return0;
916 917 if (!args_ok)
918 return2;
919 920 returnrunner.run();
921 }