Coverage for tests / scripts / test_coverage_pipeline_integration.py: 100%

1"""Integration tests for the coverage pipeline. 

2 

3Tests the full pipeline from pytest output extraction through to PR comment 

4generation, verifying that data flows correctly between components. 

5""" 

6 

7from __future__ import annotations 

8 

9import json 

10import subprocess 

11import tempfile 

12from pathlib import Path 

13 

14from assertpy import assert_that 

15 

16# Compute repo root from this test file location (tests/scripts/test_*.py -> repo root) 

17_REPO_ROOT = Path(__file__).resolve().parent.parent.parent 

18EXTRACT_SCRIPT = (_REPO_ROOT / "scripts/ci/testing/extract-test-summary.sh").resolve() 

19COMMENT_SCRIPT = (_REPO_ROOT / "scripts/ci/github/coverage-pr-comment.sh").resolve() 

20 

21 

22def test_full_pipeline_extract_to_comment() -> None: 

23 """Test the full pipeline from pytest output to JSON summary. 

24 

25 This integration test verifies that: 

26 1. extract-test-summary.sh correctly parses pytest output 

27 2. The generated JSON is valid and contains expected fields 

28 3. The JSON structure is compatible with coverage-pr-comment.sh parsing 

29 

30 Note: We don't test coverage-pr-comment.sh execution directly since it 

31 requires GitHub Actions environment variables. Instead, we verify the 

32 JSON output format is correct for downstream consumption. 

33 """ 

34 pytest_output = """ 

35============================= test session starts ============================== 

36platform linux -- Python 3.13.1, pytest-8.0.0, pluggy-1.4.0 

37rootdir: /app 

38configfile: pyproject.toml 

39plugins: cov-5.0.0, anyio-4.0.0 

40collected 156 items 

41 

42tests/unit/test_core.py .................................................. [ 32%] 

43tests/unit/test_utils.py .................................................. [ 64%] 

44tests/integration/test_api.py ............................................. [ 96%] 

45tests/integration/test_cli.py ...... [100%] 

46 

47---------- coverage: platform linux, python 3.13.1 ---------- 

48Name Stmts Miss Cover 

49--------------------------------------------- 

50lintro/__init__.py 5 0 100% 

51lintro/core.py 150 15 90% 

52lintro/utils.py 80 10 88% 

53--------------------------------------------- 

54TOTAL 235 25 89% 

55 

56============== 150 passed, 4 failed, 2 skipped in 45.67s ================ 

57""" 

58 

59 coverage_xml = """<?xml version="1.0" ?> 

60<coverage version="7.0.0" timestamp="1706472000" lines-covered="210" lines-valid="235" line-rate="0.8936" branch-rate="0" complexity="0"> 

61 <packages> 

62 <package name="lintro" line-rate="0.8936" branch-rate="0" complexity="0"> 

63 <classes> 

64 <class name="__init__.py" filename="lintro/__init__.py" line-rate="1.0" branch-rate="0" complexity="0"> 

65 <lines/> 

66 </class> 

67 <class name="core.py" filename="lintro/core.py" line-rate="0.9" branch-rate="0" complexity="0"> 

68 <lines/> 

69 </class> 

70 <class name="utils.py" filename="lintro/utils.py" line-rate="0.875" branch-rate="0" complexity="0"> 

71 <lines/> 

72 </class> 

73 </classes> 

74 </package> 

75 </packages> 

76</coverage> 

77""" 

78 

79 with tempfile.TemporaryDirectory() as tmpdir: 

80 # Set up test files 

81 input_file = Path(tmpdir) / "test-output.log" 

82 output_file = Path(tmpdir) / "test-summary.json" 

83 coverage_file = Path(tmpdir) / "coverage.xml" 

84 

85 input_file.write_text(pytest_output) 

86 coverage_file.write_text(coverage_xml) 

87 

88 # Run extract-test-summary.sh 

89 result = subprocess.run( 

90 [str(EXTRACT_SCRIPT), str(input_file), str(output_file)], 

91 capture_output=True, 

92 text=True, 

93 cwd=tmpdir, 

94 ) 

95 

96 # Verify extraction succeeded 

97 assert_that(result.returncode).is_equal_to(0) 

98 assert_that(output_file.exists()).is_true() 

99 

100 # Parse and validate JSON structure 

101 summary = json.loads(output_file.read_text()) 

102 

103 # Verify test summary fields 

104 assert_that(summary).contains_key("tests") 

105 tests = summary["tests"] 

106 assert_that(tests).contains_key("passed") 

107 assert_that(tests).contains_key("failed") 

108 assert_that(tests).contains_key("skipped") 

109 assert_that(tests).contains_key("errors") 

110 assert_that(tests).contains_key("total") 

111 assert_that(tests).contains_key("duration") 

112 

113 # Verify test values 

114 assert_that(tests["passed"]).is_equal_to(150) 

115 assert_that(tests["failed"]).is_equal_to(4) 

116 assert_that(tests["skipped"]).is_equal_to(2) 

117 assert_that(tests["duration"]).is_equal_to(45.67) 

118 

119 # Verify coverage fields 

120 assert_that(summary).contains_key("coverage") 

121 coverage = summary["coverage"] 

122 assert_that(coverage).contains_key("percentage") 

123 assert_that(coverage).contains_key("lines_covered") 

124 assert_that(coverage).contains_key("lines_total") 

125 assert_that(coverage).contains_key("lines_missing") 

126 assert_that(coverage).contains_key("files") 

127 

128 # Verify coverage values from XML 

129 assert_that(coverage["lines_covered"]).is_equal_to(210) 

130 assert_that(coverage["lines_total"]).is_equal_to(235) 

131 assert_that(coverage["lines_missing"]).is_equal_to(25) 

132 assert_that(coverage["files"]).is_equal_to(3) 

133 

134 # Verify JSON format is compatible with grep-based parsing 

135 # (single space after colon) 

136 raw_json = output_file.read_text() 

137 assert_that(raw_json).contains('"passed": ') 

138 assert_that(raw_json).contains('"failed": ') 

139 assert_that(raw_json).contains('"percentage": ') 

140 

141 

142def test_pipeline_handles_missing_coverage_xml() -> None: 

143 """Pipeline should work without coverage.xml, defaulting coverage to 0.""" 

144 pytest_output = "10 passed in 1.00s\n" 

145 

146 with tempfile.TemporaryDirectory() as tmpdir: 

147 input_file = Path(tmpdir) / "test-output.log" 

148 output_file = Path(tmpdir) / "test-summary.json" 

149 

150 input_file.write_text(pytest_output) 

151 # Note: no coverage.xml created 

152 

153 result = subprocess.run( 

154 [str(EXTRACT_SCRIPT), str(input_file), str(output_file)], 

155 capture_output=True, 

156 text=True, 

157 cwd=tmpdir, 

158 ) 

159 

160 assert_that(result.returncode).is_equal_to(0) 

161 

162 summary = json.loads(output_file.read_text()) 

163 assert_that(summary["tests"]["passed"]).is_equal_to(10) 

164 assert_that(summary["coverage"]["percentage"]).is_equal_to(0) 

165 assert_that(summary["coverage"]["files"]).is_equal_to(0) 

166 

167 

168def test_pipeline_quiet_mode_produces_valid_json() -> None: 

169 """Pipeline in quiet mode should still produce valid JSON output.""" 

170 pytest_output = "5 passed, 1 failed in 2.50s\n" 

171 

172 with tempfile.TemporaryDirectory() as tmpdir: 

173 input_file = Path(tmpdir) / "test-output.log" 

174 output_file = Path(tmpdir) / "test-summary.json" 

175 

176 input_file.write_text(pytest_output) 

177 

178 result = subprocess.run( 

179 [str(EXTRACT_SCRIPT), "--quiet", str(input_file), str(output_file)], 

180 capture_output=True, 

181 text=True, 

182 cwd=tmpdir, 

183 ) 

184 

185 assert_that(result.returncode).is_equal_to(0) 

186 # Quiet mode should suppress stdout 

187 assert_that(result.stdout).is_empty() 

188 

189 # But JSON should still be valid 

190 summary = json.loads(output_file.read_text()) 

191 assert_that(summary["tests"]["passed"]).is_equal_to(5) 

192 assert_that(summary["tests"]["failed"]).is_equal_to(1)