Coverage for tests / unit / ai / test_prompts.py: 100%

1"""Tests for AI prompt templates.""" 

2 

3from __future__ import annotations 

4 

5import re 

6 

7import pytest 

8from assertpy import assert_that 

9 

10from lintro.ai.prompts import ( 

11 FIX_BATCH_PROMPT_TEMPLATE, 

12 FIX_PROMPT_TEMPLATE, 

13 FIX_SYSTEM, 

14 POST_FIX_SUMMARY_PROMPT_TEMPLATE, 

15 REFINEMENT_PROMPT_TEMPLATE, 

16 SUMMARY_PROMPT_TEMPLATE, 

17 SUMMARY_SYSTEM, 

18) 

19 

20# Regex matching un-interpolated single-brace placeholders like {var} but not 

21# escaped double-braces like {{ or }}. 

22_LEFTOVER_PLACEHOLDER = re.compile(r"(?<!\{)\{[a-z_]+\}(?!\})") 

23 

24 

25# --------------------------------------------------------------------------- 

26# FIX_SYSTEM 

27# --------------------------------------------------------------------------- 

28 

29 

30def test_fix_system_is_non_empty(): 

31 """FIX_SYSTEM must be a non-empty string.""" 

32 assert_that(FIX_SYSTEM).is_not_empty() 

33 

34 

35def test_fix_system_mentions_json(): 

36 """FIX_SYSTEM instructs the model to respond with JSON.""" 

37 assert_that(FIX_SYSTEM).contains("JSON") 

38 

39 

40# --------------------------------------------------------------------------- 

41# FIX_PROMPT_TEMPLATE — basic rendering 

42# --------------------------------------------------------------------------- 

43 

44_FIX_DEFAULTS = { 

45 "tool_name": "ruff", 

46 "code": "E501", 

47 "file": "main.py", 

48 "line": 42, 

49 "message": "Line too long", 

50 "context_start": 37, 

51 "context_end": 47, 

52 "code_context": "x = 1", 

53 "boundary": "CODE_BLOCK_test1234", 

54} 

55 

56 

57def test_prompts_template_renders(): 

58 """Verify the fix prompt template renders with all required placeholders.""" 

59 assert_that(FIX_SYSTEM).is_not_empty() 

60 result = FIX_PROMPT_TEMPLATE.format(**_FIX_DEFAULTS) 

61 assert_that(result).contains("ruff") 

62 assert_that(result).contains("main.py") 

63 

64 

65def test_prompts_template_includes_risk_level(): 

66 """Verify the fix prompt template contains a risk_level placeholder.""" 

67 assert_that(FIX_PROMPT_TEMPLATE).contains("risk_level") 

68 

69 

70def test_fix_prompt_no_leftover_placeholders(): 

71 """All placeholders in FIX_PROMPT_TEMPLATE are interpolated.""" 

72 result = FIX_PROMPT_TEMPLATE.format(**_FIX_DEFAULTS) 

73 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

74 

75 

76def test_fix_prompt_contains_all_values(): 

77 """Every supplied value appears verbatim in the rendered prompt.""" 

78 result = FIX_PROMPT_TEMPLATE.format(**_FIX_DEFAULTS) 

79 for value in ( 

80 "ruff", 

81 "E501", 

82 "main.py", 

83 "42", 

84 "Line too long", 

85 "37", 

86 "47", 

87 "x = 1", 

88 ): 

89 assert_that(result).contains(value) 

90 

91 

92# --------------------------------------------------------------------------- 

93# FIX_PROMPT_TEMPLATE — various issue types 

94# --------------------------------------------------------------------------- 

95 

96 

97@pytest.mark.parametrize( 

98 ("tool_name", "code", "message"), 

99 [ 

100 ("ruff", "E501", "Line too long (120 > 79 characters)"), 

101 ("mypy", "attr-defined", "Module has no attribute 'foo'"), 

102 ("pylint", "C0114", "Missing module docstring"), 

103 ("flake8", "F401", "'os' imported but unused"), 

104 ("eslint", "no-unused-vars", "'x' is defined but never used"), 

105 ], 

106) 

107def test_fix_prompt_renders_various_issue_types(tool_name, code, message): 

108 """FIX_PROMPT_TEMPLATE renders correctly for diverse tool/code combos.""" 

109 result = FIX_PROMPT_TEMPLATE.format( 

110 tool_name=tool_name, 

111 code=code, 

112 file="src/app.py", 

113 line=10, 

114 message=message, 

115 context_start=5, 

116 context_end=15, 

117 code_context="pass", 

118 boundary="CODE_BLOCK_test1234", 

119 ) 

120 assert_that(result).contains(tool_name) 

121 assert_that(result).contains(code) 

122 assert_that(result).contains(message) 

123 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

124 

125 

126# --------------------------------------------------------------------------- 

127# Special characters in messages and file paths 

128# --------------------------------------------------------------------------- 

129 

130 

131def test_fix_prompt_special_characters_in_message(): 

132 """Quotes, newlines, and backslashes in the message survive rendering.""" 

133 msg = "Expected \"int\" but got 'str'\nDetails: see C:\\path" 

134 result = FIX_PROMPT_TEMPLATE.format( 

135 **{**_FIX_DEFAULTS, "message": msg}, 

136 ) 

137 assert_that(result).contains('"int"') 

138 assert_that(result).contains("'str'") 

139 assert_that(result).contains("C:\\path") 

140 

141 

142def test_fix_prompt_unicode_in_file_path(): 

143 """Unicode characters in file paths are preserved.""" 

144 path = "src/modulos/\u00e9l\u00e8ve.py" 

145 result = FIX_PROMPT_TEMPLATE.format( 

146 **{**_FIX_DEFAULTS, "file": path}, 

147 ) 

148 assert_that(result).contains(path) 

149 

150 

151def test_fix_prompt_spaces_in_file_path(): 

152 """File paths containing spaces are preserved.""" 

153 path = "my project/src/hello world.py" 

154 result = FIX_PROMPT_TEMPLATE.format( 

155 **{**_FIX_DEFAULTS, "file": path}, 

156 ) 

157 assert_that(result).contains(path) 

158 

159 

160# --------------------------------------------------------------------------- 

161# Edge cases: empty code context, zero line, very long message 

162# --------------------------------------------------------------------------- 

163 

164 

165def test_fix_prompt_empty_code_context(): 

166 """Empty code_context still produces a valid rendered string.""" 

167 result = FIX_PROMPT_TEMPLATE.format( 

168 **{**_FIX_DEFAULTS, "code_context": ""}, 

169 ) 

170 assert_that(result).is_not_empty() 

171 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

172 

173 

174def test_fix_prompt_zero_line_number(): 

175 """Line number 0 renders without error.""" 

176 result = FIX_PROMPT_TEMPLATE.format( 

177 **{**_FIX_DEFAULTS, "line": 0}, 

178 ) 

179 assert_that(result).contains("Line: 0") 

180 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

181 

182 

183def test_fix_prompt_very_long_message(): 

184 """A very long message does not break rendering.""" 

185 long_msg = "A" * 10_000 

186 result = FIX_PROMPT_TEMPLATE.format( 

187 **{**_FIX_DEFAULTS, "message": long_msg}, 

188 ) 

189 assert_that(result).contains(long_msg) 

190 

191 

192# --------------------------------------------------------------------------- 

193# SUMMARY_PROMPT_TEMPLATE and SUMMARY_SYSTEM 

194# --------------------------------------------------------------------------- 

195 

196 

197def test_summary_system_is_non_empty(): 

198 """SUMMARY_SYSTEM must be a non-empty string.""" 

199 assert_that(SUMMARY_SYSTEM).is_not_empty() 

200 

201 

202def test_summary_prompt_renders(): 

203 """SUMMARY_PROMPT_TEMPLATE renders with all required variables.""" 

204 result = SUMMARY_PROMPT_TEMPLATE.format( 

205 total_issues=42, 

206 tool_count=3, 

207 issues_digest="ruff: E501 x 10", 

208 ) 

209 assert_that(result).contains("42") 

210 assert_that(result).contains("3") 

211 assert_that(result).contains("ruff: E501 x 10") 

212 

213 

214def test_summary_prompt_no_leftover_placeholders(): 

215 """All placeholders are interpolated in SUMMARY_PROMPT_TEMPLATE.""" 

216 result = SUMMARY_PROMPT_TEMPLATE.format( 

217 total_issues=0, 

218 tool_count=0, 

219 issues_digest="", 

220 ) 

221 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

222 

223 

224def test_summary_prompt_recommends_lintro(): 

225 """SUMMARY_PROMPT_TEMPLATE tells the model to recommend lintro commands.""" 

226 assert_that(SUMMARY_PROMPT_TEMPLATE).contains("lintro chk") 

227 assert_that(SUMMARY_PROMPT_TEMPLATE).contains("lintro fmt") 

228 

229 

230# --------------------------------------------------------------------------- 

231# REFINEMENT_PROMPT_TEMPLATE 

232# --------------------------------------------------------------------------- 

233 

234 

235def test_refinement_prompt_renders(): 

236 """REFINEMENT_PROMPT_TEMPLATE renders with all required variables.""" 

237 result = REFINEMENT_PROMPT_TEMPLATE.format( 

238 tool_name="ruff", 

239 code="E501", 

240 file="main.py", 

241 line=10, 

242 previous_suggestion="old fix", 

243 new_error="still too long", 

244 context_start=5, 

245 context_end=15, 

246 code_context="x = 1", 

247 boundary="CODE_BLOCK_test1234", 

248 ) 

249 assert_that(result).contains("old fix") 

250 assert_that(result).contains("still too long") 

251 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

252 

253 

254# --------------------------------------------------------------------------- 

255# FIX_BATCH_PROMPT_TEMPLATE 

256# --------------------------------------------------------------------------- 

257 

258 

259def test_batch_prompt_renders(): 

260 """FIX_BATCH_PROMPT_TEMPLATE renders with all required variables.""" 

261 result = FIX_BATCH_PROMPT_TEMPLATE.format( 

262 tool_name="ruff", 

263 file="app.py", 

264 issues_list="1. E501 line 10\n2. E302 line 20", 

265 file_content="import os\n", 

266 boundary="CODE_BLOCK_test1234", 

267 ) 

268 assert_that(result).contains("ruff") 

269 assert_that(result).contains("app.py") 

270 assert_that(result).contains("E501 line 10") 

271 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

272 

273 

274# --------------------------------------------------------------------------- 

275# POST_FIX_SUMMARY_PROMPT_TEMPLATE 

276# --------------------------------------------------------------------------- 

277 

278 

279def test_post_fix_summary_prompt_renders(): 

280 """POST_FIX_SUMMARY_PROMPT_TEMPLATE renders with all required variables.""" 

281 result = POST_FIX_SUMMARY_PROMPT_TEMPLATE.format( 

282 applied=5, 

283 rejected=2, 

284 remaining=3, 

285 issues_digest="mypy: attr-defined x 3", 

286 ) 

287 assert_that(result).contains("5") 

288 assert_that(result).contains("2") 

289 assert_that(result).contains("3") 

290 assert_that(result).contains("mypy: attr-defined x 3") 

291 

292 

293def test_post_fix_summary_prompt_no_leftover_placeholders(): 

294 """All placeholders are interpolated in POST_FIX_SUMMARY_PROMPT_TEMPLATE.""" 

295 result = POST_FIX_SUMMARY_PROMPT_TEMPLATE.format( 

296 applied=0, 

297 rejected=0, 

298 remaining=0, 

299 issues_digest="", 

300 ) 

301 assert_that(_LEFTOVER_PLACEHOLDER.findall(result)).is_empty() 

302 

303 

304def test_post_fix_summary_recommends_lintro(): 

305 """POST_FIX_SUMMARY_PROMPT_TEMPLATE tells the model to use lintro commands.""" 

306 assert_that(POST_FIX_SUMMARY_PROMPT_TEMPLATE).contains("lintro chk") 

307 assert_that(POST_FIX_SUMMARY_PROMPT_TEMPLATE).contains("lintro fmt")