Test runner

[luke]

About
--------
This is a tool for running a program many times with varying input, and checking the output against expected values. In this way a program's correctness can be tested, (hopefully) making the programmer more confident of the absence of bugs. It can also help with confirming no bugs have been introduced later on, and that fixed bugs stay fixed.

This was written as part of the L-BASIC project, but is fairly generic and should work with any program that takes a filename in COMMAND$ and prints output to the console (i.e. use $CONSOLE:ONLY). Thanks to @Ed Davis for providing Windows compatibility patches.

Usage
---------
Let's say we have a simple program, save it as sample.bas and compile:

Code: QB64: [Select]

1. $Console:Only
2. _Dest _Console
3. Open Command$ For Input As #1
4. While Not EOF(1)
5.     Line Input #1, l$
6.     If InStr(l$, "Q") Then
7.         Print "Error, found a Q"
8.         System 1 'Exit with an error if we read a Q
9.     End If
10.     Print UCase$(l$)
11. Wend
12. System

It reads a file and prints out the contents in upper case, but requires the input not contain the letter Q: that is considered invalid input and will give an error.

Now we can write some test cases! The format should be fairly clear; the file is one or more test cases, delimited with various $commands:

Code: [Select]

$title: Letters are uppercased
the rain in spain
$expect: stdout
THE RAIN IN SPAIN
$finish

$title: Uppercased letters stay uppercased
The rain in Spain
$expect: stdout
THE RAIN IN SPAIN
$finish

$title: Multi-line input
Now is the time for
all good men to
come to the aid
of their party
$expect: stdout
NOW IS THE TIME FOR
ALL GOOD MEN TO
COME TO THE AID
OF THEIR PARTY
$finish

$title: Bad input is caught
This has a Q in it
$expect: error


The structure is:

• Each test begins with $title to give the test an identifiying name.
• The contents of the input file is provided
• $expect: stdout indicates that we should check the standard (i.e. console) output of the program
• The expected output is provided
• $finish indicates the end of the expected output
If instead an error is expected, we can $expect: error. No expected output or $finish is required then.

$expect: stdout strips leading and trailing whitespace before comparing. Use $expect: stdout_exact if you don't want that.

Now you can run it (on Mac/Linux, use './sample'):

Code: [Select]

test-runner.exe sample.exe our-tests.txt

and get an output similar to this:

Code: [Select]

test:Letters are uppercased: OK
test:Uppercased letters stay uppercased: OK
test:Multi-line input: OK
test:Bad input is caught: OK
Total 4/4 OK in 4 seconds

If any test failed, it would show us the expected and actual output to help fix the program.

Code: QB64: [Select]

1. 'Copyright 2020 Luke Ceddia
2. 'SPDX-License-Identifier: Apache-2.0
3. 'test.bas - Run test suite
4.  
5. DefLng A-Z
6. $Console:Only
7. _Dest _Console
8. Randomize Timer
9.  
10. Type test_unit_t
11.     title As String
12.     program As String
13.     expect As String
14.     expected_output As String
15. End Type
16. ReDim tests(0) As test_unit_t
17. Dim active_section '0 = none, 1 = program, 2 = output
18.  
19. On Error GoTo ehandler
20.  
21. If _CommandCount < 2 Then
22.     Print "Usage: "; Command$(0); " <test program> <test files>"
23.     System 1
24. End If
25.  
26. If InStr(_OS$, "WINDOWS") Then
27.     tmpdir$ = Environ$("TEMP") + "\"
28. Else
29.     tmpdir$ = "/tmp/"
30. End If
31.  
32. testbinary$ = Command$(1)
33. ChDir _StartDir$
34. For cmdline_index = 2 To _CommandCount
35.     Open Command$(cmdline_index) For Binary As #1
36.     While Not EOF(1)
37.         Line Input #1, l$
38.         lt$ = LTrim$(l$)
39.         If Left$(lt$, 7) = "$title:" Then
40.             active_test = UBound(tests) + 1
41.             ReDim _Preserve tests(1 To active_test) As test_unit_t
42.             tests(active_test).title = basename$(Command$(cmdline_index)) + ":" + LTrim$(Mid$(lt$, 8))
43.             active_section = 1
44.         ElseIf Left$(lt$, 8) = "$expect:" Then
45.             tests(active_test).expect = LTrim$(Mid$(lt$, 9))
46.             active_section = 2
47.         ElseIf Left$(lt$, 7) = "$finish" Then
48.             active_section = 0
49.         ElseIf active_section = 1 Then
50.             tests(active_test).program = tests(active_test).program + l$ + Chr$(10)
51.         ElseIf active_section = 2 Then
52.             tests(active_test).expected_output = tests(active_test).expected_output + l$ + Chr$(10)
53.         ElseIf lt$ = "" Or Left$(lt$, 1) = "#" Or Left$(lt$, 1) = "'" Then
54.             'Blank line or comment, do nothing
55.         Else
56.             Print "Must start with $title"
57.             System 1
58.         End If
59.     Wend
60.     Close #1
61. Next cmdline_index
62.  
63. starttime! = Timer(0.001)
64. For active_test = 1 To UBound(tests)
65.     'print "TITLE: "; tests(active_test).title
66.     'print "PROGRAM"
67.     'print "-------"
68.     'print tests(active_test).program;
69.     'print "EXPECT: "; tests(active_test).expect
70.     'print tests(active_test).expected_output;
71.  
72.     filename$ = tmpdir$ + "test-" + rndhex$(4)
73.     Open filename$ + ".bas" For Output As #1
74.     Print #1, tests(active_test).program
75.     Close #1
76.     retcode = Shell(testbinary$ + " " + filename$ + ".bas > " + filename$ + ".output")
77.     Print tests(active_test).title; ": ";
78.     Select Case tests(active_test).expect
79.         Case "error"
80.             If retcode > 0 Then
81.                 Print "OK"
82.                 successes = successes + 1
83.             Else
84.                 Print "Failed, expected error but ran successfully."
85.             End If
86.         Case "stdout", "stdout_exact"
87.             Open filename$ + ".output" For Binary As #1
88.             actual_output$ = Space$(LOF(1))
89.             Get #1, , actual_output$
90.             Close #1
91.             actual_output$ = remove_char$(actual_output$, Chr$(13))
92.             tests(active_test).expected_output = remove_char$(tests(active_test).expected_output, Chr$(13))
93.  
94.             If tests(active_test).expect <> "stdout_exact" Then
95.                 actual_output$ = strip$(actual_output$)
96.                 tests(active_test).expected_output = strip$(tests(active_test).expected_output)
97.             End If
98.  
99.             If retcode > 0 Then
100.                 Print "Failed with error, output was: "; actual_output$
101.             ElseIf actual_output$ = tests(active_test).expected_output Then
102.                 Print "OK"
103.                 successes = successes + 1
104.             Else
105.                 Print "Failed!"
106.                 Print "Expected: "; tests(active_test).expected_output;
107.                 Print "  Actual: "; actual_output$;
108.             End If
109.         Case Else
110.             Print "Unknown condition"
111.     End Select
112.     Kill filename$ + ".bas"
113.     Kill filename$ + ".output"
114. Next active_test
115. endtime! = Timer(0.001)
116.  
117. Print "Total"; Str$(successes); "/"; LTrim$(Str$(UBound(tests))); " OK in"; Int((endtime! - starttime!) * 10) / 10; "seconds"
118.  
119. System
120.  
121. ehandler:
122. Print "Error"; Err; "on line"; _ErrorLine
123. System 2
124.  
125. Function basename$ (path$)
126.     dot = _InStrRev(path$, ".")
127.     slash = _InStrRev(path$, "/")
128.     basename$ = Mid$(path$, slash + 1, dot - slash - 1)
129. End Function
130.  
131. Function rndhex$ (length)
132.     For i = 1 To length
133.         result$ = result$ + Hex$(Int(Rnd * 256))
134.     Next i
135.     rndhex$ = result$
136. End Function
137.  
138. 'Courtesy Ed Davis
139. Function remove_char$ (s$, c$)
140.     Dim s2$
141.     Dim i As Integer
142.  
143.     s2$ = ""
144.     For i = 1 To Len(s$)
145.         If Mid$(s$, i, 1) <> c$ Then
146.             s2$ = s2$ + Mid$(s$, i, 1)
147.         End If
148.     Next
149.     remove_char$ = s2$
150. End Function
151.  
152. Function strip$ (s$)
153.     start = 0
154.     whitespace$ = Chr$(10) + Chr$(9) + Chr$(32)
155.     Do
156.         start = start + 1
157.     Loop While InStr(whitespace$, Mid$(s$, start, 1))
158.     finish = Len(s$) + 1
159.     Do
160.         finish = finish - 1
161.     Loop While InStr(whitespace$, Mid$(s$, finish, 1))
162.     strip$ = Mid$(s$, start, finish - start + 1)
163. End Function
164.