Coverage for tests / scripts / test_extract_test_summary.py: 100%

1"""Tests for extract-test-summary.sh script. 

2 

3This module tests the extract-test-summary.sh script which parses pytest output 

4and extracts test results into a JSON summary file for PR comments. 

5""" 

6 

7from __future__ import annotations 

8 

9import json 

10import os 

11import subprocess 

12import tempfile 

13from pathlib import Path 

14 

15from assertpy import assert_that 

16 

17# Compute repo root from this test file location (tests/scripts/test_*.py -> repo root) 

18_REPO_ROOT = Path(__file__).resolve().parent.parent.parent 

19SCRIPT_PATH = _REPO_ROOT / "scripts/ci/testing/extract-test-summary.sh" 

20 

21 

22def test_script_help_output() -> None: 

23 """Script should display help and exit 0 with --help flag.""" 

24 result = subprocess.run( 

25 [str(SCRIPT_PATH), "--help"], 

26 capture_output=True, 

27 text=True, 

28 ) 

29 

30 assert_that(result.returncode).is_equal_to(0) 

31 assert_that(result.stdout).contains("Usage:") 

32 assert_that(result.stdout).contains("extract-test-summary.sh") 

33 assert_that(result.stdout).contains("test-output-file") 

34 assert_that(result.stdout).contains("output-json-file") 

35 

36 

37def test_script_syntax_check() -> None: 

38 """Script should pass bash syntax check.""" 

39 result = subprocess.run( 

40 ["bash", "-n", str(SCRIPT_PATH)], 

41 capture_output=True, 

42 text=True, 

43 ) 

44 

45 assert_that(result.returncode).is_equal_to(0) 

46 assert_that(result.stderr).is_empty() 

47 

48 

49def test_extract_from_standard_pytest_output() -> None: 

50 """Extract test summary from standard pytest output format.""" 

51 pytest_output = """ 

52============================= test session starts ============================== 

53platform linux -- Python 3.13.1, pytest-8.0.0, pluggy-1.4.0 

54collected 100 items 

55 

56tests/test_foo.py ............................ [ 28%] 

57tests/test_bar.py ............................ [ 56%] 

58tests/test_baz.py ............................ [ 84%] 

59tests/test_qux.py ................ [100%] 

60 

61=============================== warnings summary =============================== 

621 warning 

63 

64============== 95 passed, 3 failed, 2 skipped in 12.34s ================ 

65""" 

66 with tempfile.TemporaryDirectory() as tmpdir: 

67 input_file = Path(tmpdir) / "test-output.log" 

68 output_file = Path(tmpdir) / "test-summary.json" 

69 input_file.write_text(pytest_output) 

70 

71 result = subprocess.run( 

72 [str(SCRIPT_PATH), str(input_file), str(output_file)], 

73 capture_output=True, 

74 text=True, 

75 cwd=tmpdir, 

76 ) 

77 

78 assert_that(result.returncode).is_equal_to(0) 

79 assert_that(output_file.exists()).is_true() 

80 

81 summary = json.loads(output_file.read_text()) 

82 assert_that(summary["tests"]["passed"]).is_equal_to(95) 

83 assert_that(summary["tests"]["failed"]).is_equal_to(3) 

84 assert_that(summary["tests"]["skipped"]).is_equal_to(2) 

85 assert_that(summary["tests"]["duration"]).is_equal_to(12.34) 

86 

87 

88def test_extract_from_lintro_table_format() -> None: 

89 """Extract test summary from lintro table format output. 

90 

91 Note: The pattern supports both emoji and non-emoji formats 

92 (e.g., '| 🧪 pytest' or '| pytest'). 

93 """ 

94 lintro_output = """ 

95| Tool | Status | Passed | Failed | Skipped | Total | Duration | 

96|------|--------|--------|--------|---------|-------|----------| 

97| pytest | PASS | 150 | 0 | 5 | 155 | 45.67s | 

98""" 

99 with tempfile.TemporaryDirectory() as tmpdir: 

100 input_file = Path(tmpdir) / "test-output.log" 

101 output_file = Path(tmpdir) / "test-summary.json" 

102 input_file.write_text(lintro_output) 

103 

104 result = subprocess.run( 

105 [str(SCRIPT_PATH), str(input_file), str(output_file)], 

106 capture_output=True, 

107 text=True, 

108 cwd=tmpdir, 

109 ) 

110 

111 assert_that(result.returncode).is_equal_to(0) 

112 assert_that(output_file.exists()).is_true() 

113 

114 summary = json.loads(output_file.read_text()) 

115 assert_that(summary["tests"]["passed"]).is_equal_to(150) 

116 assert_that(summary["tests"]["failed"]).is_equal_to(0) 

117 assert_that(summary["tests"]["skipped"]).is_equal_to(5) 

118 assert_that(summary["tests"]["total"]).is_equal_to(155) 

119 

120 

121def test_extract_with_coverage_xml() -> None: 

122 """Extract both test summary and coverage data when coverage.xml exists.""" 

123 pytest_output = "10 passed in 5.00s\n" 

124 coverage_xml = """<?xml version="1.0" ?> 

125<coverage version="7.0.0" timestamp="1234567890" lines-covered="800" lines-valid="1000" line-rate="0.8" branch-rate="0.0" complexity="0"> 

126 <packages> 

127 <package name="lintro" line-rate="0.8" branch-rate="0.0" complexity="0"> 

128 <classes> 

129 <class name="foo.py" filename="lintro/foo.py" line-rate="0.9" branch-rate="0.0" complexity="0"> 

130 <lines/> 

131 </class> 

132 <class name="bar.py" filename="lintro/bar.py" line-rate="0.7" branch-rate="0.0" complexity="0"> 

133 <lines/> 

134 </class> 

135 </classes> 

136 </package> 

137 </packages> 

138</coverage> 

139""" 

140 with tempfile.TemporaryDirectory() as tmpdir: 

141 input_file = Path(tmpdir) / "test-output.log" 

142 output_file = Path(tmpdir) / "test-summary.json" 

143 coverage_file = Path(tmpdir) / "coverage.xml" 

144 

145 input_file.write_text(pytest_output) 

146 coverage_file.write_text(coverage_xml) 

147 

148 result = subprocess.run( 

149 [str(SCRIPT_PATH), str(input_file), str(output_file)], 

150 capture_output=True, 

151 text=True, 

152 cwd=tmpdir, 

153 ) 

154 

155 assert_that(result.returncode).is_equal_to(0) 

156 

157 summary = json.loads(output_file.read_text()) 

158 assert_that(summary["coverage"]["percentage"]).is_equal_to(80.0) 

159 assert_that(summary["coverage"]["lines_covered"]).is_equal_to(800) 

160 assert_that(summary["coverage"]["lines_total"]).is_equal_to(1000) 

161 assert_that(summary["coverage"]["lines_missing"]).is_equal_to(200) 

162 assert_that(summary["coverage"]["files"]).is_equal_to(2) 

163 

164 

165def test_extract_from_environment_variables() -> None: 

166 """Extract test summary from environment variables when no input file.""" 

167 env = os.environ.copy() 

168 env["TEST_PASSED"] = "42" 

169 env["TEST_FAILED"] = "1" 

170 env["TEST_SKIPPED"] = "3" 

171 env["TEST_ERRORS"] = "0" 

172 env["TEST_TOTAL"] = "46" 

173 env["TEST_DURATION"] = "8.5" 

174 

175 with tempfile.TemporaryDirectory() as tmpdir: 

176 output_file = Path(tmpdir) / "test-summary.json" 

177 

178 result = subprocess.run( 

179 [str(SCRIPT_PATH), "", str(output_file)], 

180 capture_output=True, 

181 text=True, 

182 cwd=tmpdir, 

183 env=env, 

184 ) 

185 

186 assert_that(result.returncode).is_equal_to(0) 

187 

188 summary = json.loads(output_file.read_text()) 

189 assert_that(summary["tests"]["passed"]).is_equal_to(42) 

190 assert_that(summary["tests"]["failed"]).is_equal_to(1) 

191 assert_that(summary["tests"]["skipped"]).is_equal_to(3) 

192 assert_that(summary["tests"]["total"]).is_equal_to(46) 

193 

194 

195def test_default_output_file() -> None: 

196 """Script uses test-summary.json as default output file.""" 

197 pytest_output = "5 passed in 1.00s\n" 

198 

199 with tempfile.TemporaryDirectory() as tmpdir: 

200 input_file = Path(tmpdir) / "test-output.log" 

201 input_file.write_text(pytest_output) 

202 

203 # Run without specifying output file 

204 result = subprocess.run( 

205 [str(SCRIPT_PATH), str(input_file)], 

206 capture_output=True, 

207 text=True, 

208 cwd=tmpdir, 

209 ) 

210 

211 assert_that(result.returncode).is_equal_to(0) 

212 

213 # Default output file should be created 

214 default_output = Path(tmpdir) / "test-summary.json" 

215 assert_that(default_output.exists()).is_true() 

216 

217 summary = json.loads(default_output.read_text()) 

218 assert_that(summary["tests"]["passed"]).is_equal_to(5)