1use crate::error::SsgError;
14use crate::plugin::{Plugin, PluginContext};
15use crate::util::head_dom::inject_before_head_close;
16use anyhow::Result;
17use std::{
18 collections::BTreeMap,
19 fs,
20 path::{Path, PathBuf},
21};
22
23#[derive(Debug, Clone, Copy)]
30pub struct AiPlugin;
31
32impl Plugin for AiPlugin {
33 fn name(&self) -> &'static str {
34 "ai"
35 }
36
37 fn after_compile(&self, ctx: &PluginContext) -> Result<(), SsgError> {
38 if !ctx.site_dir.exists() {
39 return Ok(());
40 }
41
42 generate_llms_txt(&ctx.site_dir, ctx.config.as_ref())
43 .map_err(|e| SsgError::io(e, &ctx.site_dir))?;
44 generate_llms_full_txt(&ctx.site_dir, ctx.config.as_ref())
45 .map_err(|e| SsgError::io(e, &ctx.site_dir))?;
46
47 let html_files = collect_html_files(&ctx.site_dir)
48 .map_err(|e| SsgError::io(e, &ctx.site_dir))?;
49 let pages_with_missing_alt =
50 process_html_for_ai(&html_files, &ctx.site_dir)
51 .map_err(|e| SsgError::io(e, &ctx.site_dir))?;
52
53 if pages_with_missing_alt > 0 {
54 log::warn!(
55 "[ai] {pages_with_missing_alt} page(s) have images without alt text"
56 );
57 }
58
59 Ok(())
60 }
61}
62
63fn process_html_for_ai(
65 html_files: &[PathBuf],
66 site_dir: &Path,
67) -> Result<usize> {
68 let mut pages_with_missing_alt = 0usize;
69
70 for path in html_files {
71 let html = fs::read_to_string(path)?;
72 let modified = inject_max_snippet(&html);
73
74 check_alt_text(path, &modified, site_dir, &mut pages_with_missing_alt);
75
76 if modified != html {
77 fs::write(path, modified)?;
78 }
79 }
80
81 Ok(pages_with_missing_alt)
82}
83
84fn inject_max_snippet(html: &str) -> String {
86 if html.contains("max-snippet") {
87 return html.to_string();
88 }
89 let tag = "<meta name=\"robots\" content=\"max-snippet:-1, max-image-preview:large, max-video-preview:-1\">\n";
90 inject_before_head_close(html, tag)
91}
92
93fn check_alt_text(
95 path: &Path,
96 html: &str,
97 site_dir: &Path,
98 counter: &mut usize,
99) {
100 let missing = count_missing_alt(html);
101 if missing > 0 {
102 let rel = path.strip_prefix(site_dir).unwrap_or(path).display();
103 log::warn!("[ai] {missing} image(s) missing alt text in {rel}");
104 *counter += 1;
105 }
106}
107
108fn collect_page_entries(
117 site_dir: &Path,
118) -> Result<Vec<(String, String, String)>> {
119 let html_files = collect_html_files(site_dir)?;
120 let mut entries = Vec::new();
121
122 for html_path in &html_files {
123 let rel = html_path.strip_prefix(site_dir).unwrap_or(html_path);
124
125 let sidecar_path = html_path.with_extension("meta.json");
127 let meta: serde_json::Map<String, serde_json::Value> =
128 if sidecar_path.exists() {
129 if let Ok(content) = fs::read_to_string(&sidecar_path) {
130 serde_json::from_str(&content).unwrap_or_default()
131 } else {
132 serde_json::Map::new()
133 }
134 } else {
135 serde_json::Map::new()
136 };
137
138 if is_excluded_page(rel, &meta) {
139 continue;
140 }
141
142 let title = meta
143 .get("title")
144 .and_then(serde_json::Value::as_str)
145 .unwrap_or_default()
146 .to_string();
147 let description = meta
148 .get("description")
149 .and_then(serde_json::Value::as_str)
150 .unwrap_or_default()
151 .to_string();
152
153 let url = format!("/{}", rel.to_string_lossy().replace('\\', "/"));
155
156 if !title.is_empty() {
157 entries.push((title, url, description));
158 }
159 }
160
161 Ok(entries)
162}
163
164fn is_excluded_page(
168 path: &Path,
169 frontmatter: &serde_json::Map<String, serde_json::Value>,
170) -> bool {
171 let file_name = path
173 .file_name()
174 .map(|n| n.to_string_lossy().to_lowercase())
175 .unwrap_or_default();
176 if file_name == "404.html" || file_name.starts_with("error") {
177 return true;
178 }
179
180 if let Some(draft) = frontmatter.get("draft") {
182 if draft.as_bool().unwrap_or(false)
183 || draft.as_str().is_some_and(|s| s == "true")
184 {
185 return true;
186 }
187 }
188
189 if let Some(private) = frontmatter.get("private") {
191 if private.as_bool().unwrap_or(false)
192 || private.as_str().is_some_and(|s| s == "true")
193 {
194 return true;
195 }
196 }
197
198 false
199}
200
201fn group_pages_by_section(
206 entries: &[(String, String, String)],
207) -> BTreeMap<String, Vec<(String, String, String)>> {
208 let mut sections: BTreeMap<String, Vec<(String, String, String)>> =
209 BTreeMap::new();
210
211 for (title, url, description) in entries {
212 let trimmed = url.trim_start_matches('/');
214 let section = if let Some(slash) = trimmed.find('/') {
215 let dir = &trimmed[..slash];
216 titlecase_word(dir)
217 } else {
218 "Pages".to_string()
219 };
220
221 sections.entry(section).or_default().push((
222 title.clone(),
223 url.clone(),
224 description.clone(),
225 ));
226 }
227
228 sections
229}
230
231fn titlecase_word(s: &str) -> String {
233 let mut chars = s.chars();
234 match chars.next() {
235 None => String::new(),
236 Some(first) => {
237 let upper: String = first.to_uppercase().collect();
238 format!("{upper}{}", chars.as_str().to_lowercase())
239 }
240 }
241}
242
243fn parse_robots_disallow(site_dir: &Path) -> Vec<String> {
245 let robots_path = site_dir.join("robots.txt");
246 let Ok(content) = fs::read_to_string(&robots_path) else {
247 return Vec::new();
248 };
249
250 content
251 .lines()
252 .filter_map(|line| {
253 let trimmed = line.trim();
254 if let Some(rest) = trimmed.strip_prefix("Disallow:") {
255 let pattern = rest.trim();
256 if !pattern.is_empty() {
257 return Some(pattern.to_string());
258 }
259 }
260 None
261 })
262 .collect()
263}
264
265fn generate_llms_txt(
282 site_dir: &Path,
283 config: Option<&crate::cmd::SsgConfig>,
284) -> Result<()> {
285 let site_name = config.map_or("Site", |c| c.site_name.as_str());
286 let base_url = config.map_or("", |c| c.base_url.as_str());
287 let description = config.map_or("", |c| c.site_description.as_str());
288 let language = config
289 .map(|c| c.language.as_str())
290 .filter(|l| !l.is_empty())
291 .unwrap_or("en");
292 let canonical_root = base_url.trim_end_matches('/');
293
294 let mut content =
295 format!("# {site_name}\n\n> {description}\n\nLanguage: {language}\n");
296
297 let entries = collect_page_entries(site_dir).unwrap_or_default();
299 let sections = group_pages_by_section(&entries);
300
301 for (section, pages) in §ions {
302 content.push_str(&format!("\n## {section}\n"));
303 for (title, url, desc) in pages {
304 let full_url = if canonical_root.is_empty() {
305 url.clone()
306 } else {
307 format!("{canonical_root}{url}")
308 };
309 if desc.is_empty() {
310 content.push_str(&format!("- [{title}]({full_url})\n"));
311 } else {
312 content.push_str(&format!("- [{title}]({full_url}): {desc}\n"));
313 }
314 }
315 }
316
317 let disallow = parse_robots_disallow(site_dir);
319 if !disallow.is_empty() {
320 content.push_str("\n## Disallow\n");
321 for pattern in &disallow {
322 content.push_str(&format!("- {pattern}\n"));
323 }
324 }
325
326 fs::write(site_dir.join("llms.txt"), content)?;
327 log::info!("[ai] Generated llms.txt");
328 Ok(())
329}
330
331fn generate_llms_full_txt(
336 site_dir: &Path,
337 config: Option<&crate::cmd::SsgConfig>,
338) -> Result<()> {
339 let site_name = config.map_or("Site", |c| c.site_name.as_str());
340 let base_url = config.map_or("", |c| c.base_url.as_str());
341 let description = config.map_or("", |c| c.site_description.as_str());
342 let language = config
343 .map(|c| c.language.as_str())
344 .filter(|l| !l.is_empty())
345 .unwrap_or("en");
346 let canonical_root = base_url.trim_end_matches('/');
347
348 let mut content =
349 format!("# {site_name}\n\n> {description}\n\nLanguage: {language}\n");
350
351 let html_files = collect_html_files(site_dir)?;
352
353 for html_path in &html_files {
354 let rel = html_path.strip_prefix(site_dir).unwrap_or(html_path);
355
356 let sidecar_path = html_path.with_extension("meta.json");
358 let meta: serde_json::Map<String, serde_json::Value> =
359 if sidecar_path.exists() {
360 if let Ok(c) = fs::read_to_string(&sidecar_path) {
361 serde_json::from_str(&c).unwrap_or_default()
362 } else {
363 serde_json::Map::new()
364 }
365 } else {
366 serde_json::Map::new()
367 };
368
369 if is_excluded_page(rel, &meta) {
370 continue;
371 }
372
373 let title = meta
374 .get("title")
375 .and_then(serde_json::Value::as_str)
376 .unwrap_or_default();
377
378 if title.is_empty() {
379 continue;
380 }
381
382 let url = format!("/{}", rel.to_string_lossy().replace('\\', "/"));
383 let full_url = if canonical_root.is_empty() {
384 url.clone()
385 } else {
386 format!("{canonical_root}{url}")
387 };
388
389 let html = fs::read_to_string(html_path).unwrap_or_default();
391 let body_text = strip_html_tags(&extract_body(&html));
392 let trimmed = collapse_whitespace(&body_text);
393
394 content.push_str(&format!("\n---\n\n## [{title}]({full_url})\n\n"));
395 if !trimmed.is_empty() {
396 content.push_str(&trimmed);
397 content.push('\n');
398 }
399 }
400
401 fs::write(site_dir.join("llms-full.txt"), content)?;
402 log::info!("[ai] Generated llms-full.txt");
403 Ok(())
404}
405
406fn extract_body(html: &str) -> String {
408 let lower = html.to_lowercase();
409 let start = lower
410 .find("<body")
411 .and_then(|i| lower[i..].find('>').map(|j| i + j + 1))
412 .unwrap_or(0);
413 let end = lower.find("</body>").unwrap_or(html.len());
414 html[start..end].to_string()
415}
416
417fn strip_html_tags(html: &str) -> String {
419 let mut result = String::with_capacity(html.len());
420 let mut in_tag = false;
421 for ch in html.chars() {
422 match ch {
423 '<' => in_tag = true,
424 '>' => in_tag = false,
425 _ if !in_tag => result.push(ch),
426 _ => {}
427 }
428 }
429 result
430}
431
432fn collapse_whitespace(s: &str) -> String {
434 let mut result = String::with_capacity(s.len());
435 let mut prev_ws = true; for ch in s.chars() {
437 if ch.is_whitespace() {
438 if !prev_ws {
439 result.push(' ');
440 prev_ws = true;
441 }
442 } else {
443 result.push(ch);
444 prev_ws = false;
445 }
446 }
447 if result.ends_with(' ') {
449 let _ = result.pop();
450 }
451 result
452}
453
454fn count_missing_alt(html: &str) -> usize {
456 let lower = html.to_lowercase();
457 let mut count = 0;
458 let mut pos = 0;
459 while let Some(start) = lower[pos..].find("<img") {
460 let abs = pos + start;
461 let tag_end =
462 lower[abs..].find('>').map_or(lower.len(), |e| abs + e + 1);
463 let tag = &lower[abs..tag_end];
464
465 let has_alt = tag.contains("alt=");
466 let empty_alt = tag.contains("alt=\"\"") || tag.contains("alt=''");
467 if !has_alt || empty_alt {
468 count += 1;
469 }
470 pos = tag_end;
471 }
472 count
473}
474
475fn collect_html_files(dir: &Path) -> Result<Vec<PathBuf>> {
477 fail_point!("ai::collect-html-files", |_| {
478 Err(anyhow::anyhow!("injected: ai::collect-html-files"))
479 });
480 crate::walk::walk_files(dir, "html").map_err(Into::into)
481}
482
483#[cfg(test)]
484mod tests {
485 #![allow(clippy::unwrap_used, clippy::expect_used)]
486
487 use super::*;
488 use crate::cmd::SsgConfig;
489 use crate::test_support::init_logger;
490 use std::path::PathBuf;
491 use tempfile::{tempdir, TempDir};
492
493 fn make_site() -> (TempDir, PathBuf, PluginContext) {
498 init_logger();
499 let dir = tempdir().expect("create tempdir");
500 let site = dir.path().join("site");
501 fs::create_dir_all(&site).expect("mkdir site");
502 let ctx = PluginContext::new(dir.path(), dir.path(), &site, dir.path());
503 (dir, site, ctx)
504 }
505
506 fn write_page(
508 site: &Path,
509 rel_path: &str,
510 title: &str,
511 description: &str,
512 extra_fields: &str,
513 ) {
514 let html_path = site.join(rel_path);
515 if let Some(parent) = html_path.parent() {
516 fs::create_dir_all(parent).unwrap();
517 }
518 let html = format!(
519 "<html><head><title>{title}</title></head>\
520 <body><h1>{title}</h1><p>{description}</p></body></html>"
521 );
522 fs::write(&html_path, html).unwrap();
523
524 let mut sidecar_json =
525 format!(r#"{{"title": "{title}", "description": "{description}""#);
526 if !extra_fields.is_empty() {
527 sidecar_json.push_str(", ");
528 sidecar_json.push_str(extra_fields);
529 }
530 sidecar_json.push('}');
531 fs::write(html_path.with_extension("meta.json"), sidecar_json).unwrap();
532 }
533
534 #[test]
539 fn ai_plugin_is_copy_after_move() {
540 let plugin = AiPlugin;
542 let _copy = plugin;
543 assert_eq!(plugin.name(), "ai");
544 }
545
546 #[test]
547 fn name_returns_static_ai_identifier() {
548 assert_eq!(AiPlugin.name(), "ai");
549 }
550
551 #[test]
556 fn count_missing_alt_table_driven() {
557 let cases: &[(&str, usize, &str)] = &[
558 (
560 r#"<img src="a.jpg" alt="ok">"#,
561 0,
562 "alt present and non-empty",
563 ),
564 (r#"<img src="a.jpg">"#, 1, "no alt attribute at all"),
565 (r#"<img src="a.jpg" alt="">"#, 1, "empty double-quoted alt"),
566 (r#"<img src="a.jpg" alt=''>"#, 1, "empty single-quoted alt"),
567 (
568 r#"<img src="a.jpg"><img src="b.jpg" alt="ok">"#,
569 1,
570 "first missing, second ok",
571 ),
572 (
573 r#"<img src="a.jpg"><img src="b.jpg">"#,
574 2,
575 "both missing — sequential scan progresses",
576 ),
577 ("", 0, "empty input → zero"),
578 ("<p>no images here</p>", 0, "no <img> tags at all"),
579 (r#"<IMG SRC="a.jpg" ALT="ok">"#, 0, "case-insensitive ALT"),
580 (r#"<IMG SRC="a.jpg">"#, 1, "uppercase tag, no alt"),
581 ];
582 for (input, expected, comment) in cases {
583 assert_eq!(
584 count_missing_alt(input),
585 *expected,
586 "{comment}: count_missing_alt({input:?})"
587 );
588 }
589 }
590
591 #[test]
592 fn count_missing_alt_unterminated_tag_does_not_panic() {
593 let result = count_missing_alt("<img src=foo");
594 assert!(result <= 1);
595 }
596
597 #[test]
602 fn test_parse_robots_disallow() {
603 let dir = tempdir().expect("tempdir");
604
605 fs::write(
607 dir.path().join("robots.txt"),
608 "User-agent: *\nDisallow: /admin/\nDisallow: /private/\nAllow: /\n",
609 )
610 .unwrap();
611 let result = parse_robots_disallow(dir.path());
612 assert_eq!(result, vec!["/admin/", "/private/"]);
613 }
614
615 #[test]
616 fn test_parse_robots_disallow_empty_file() {
617 let dir = tempdir().expect("tempdir");
618 fs::write(dir.path().join("robots.txt"), "").unwrap();
619 let result = parse_robots_disallow(dir.path());
620 assert!(result.is_empty());
621 }
622
623 #[test]
624 fn test_parse_robots_disallow_no_disallow_lines() {
625 let dir = tempdir().expect("tempdir");
626 fs::write(
627 dir.path().join("robots.txt"),
628 "User-agent: *\nAllow: /\nSitemap: https://example.com/sitemap.xml\n",
629 )
630 .unwrap();
631 let result = parse_robots_disallow(dir.path());
632 assert!(result.is_empty());
633 }
634
635 #[test]
636 fn test_parse_robots_disallow_multiple_user_agents() {
637 let dir = tempdir().expect("tempdir");
638 fs::write(
639 dir.path().join("robots.txt"),
640 "User-agent: Googlebot\nDisallow: /nogoogle/\n\n\
641 User-agent: *\nDisallow: /secret/\n",
642 )
643 .unwrap();
644 let result = parse_robots_disallow(dir.path());
645 assert_eq!(result, vec!["/nogoogle/", "/secret/"]);
646 }
647
648 #[test]
649 fn test_parse_robots_disallow_missing_file() {
650 let dir = tempdir().expect("tempdir");
651 let result = parse_robots_disallow(dir.path());
652 assert!(result.is_empty());
653 }
654
655 #[test]
656 fn test_parse_robots_disallow_empty_pattern_skipped() {
657 let dir = tempdir().expect("tempdir");
659 fs::write(
660 dir.path().join("robots.txt"),
661 "User-agent: *\nDisallow:\nDisallow: /blocked/\n",
662 )
663 .unwrap();
664 let result = parse_robots_disallow(dir.path());
665 assert_eq!(result, vec!["/blocked/"]);
666 }
667
668 #[test]
673 fn test_is_excluded_page_draft() {
674 let mut meta = serde_json::Map::new();
675 let _ = meta.insert("draft".to_string(), serde_json::Value::Bool(true));
676 assert!(is_excluded_page(Path::new("post.html"), &meta));
677 }
678
679 #[test]
680 fn test_is_excluded_page_draft_string() {
681 let mut meta = serde_json::Map::new();
682 let _ = meta.insert(
683 "draft".to_string(),
684 serde_json::Value::String("true".to_string()),
685 );
686 assert!(is_excluded_page(Path::new("post.html"), &meta));
687 }
688
689 #[test]
690 fn test_is_excluded_page_private() {
691 let mut meta = serde_json::Map::new();
692 let _ =
693 meta.insert("private".to_string(), serde_json::Value::Bool(true));
694 assert!(is_excluded_page(Path::new("post.html"), &meta));
695 }
696
697 #[test]
698 fn test_is_excluded_page_404() {
699 let meta = serde_json::Map::new();
700 assert!(is_excluded_page(Path::new("404.html"), &meta));
701 }
702
703 #[test]
704 fn test_is_excluded_page_normal() {
705 let mut meta = serde_json::Map::new();
706 let _ = meta.insert(
707 "title".to_string(),
708 serde_json::Value::String("Hello".to_string()),
709 );
710 assert!(!is_excluded_page(Path::new("index.html"), &meta));
711 }
712
713 #[test]
714 fn test_is_excluded_page_error_page() {
715 let meta = serde_json::Map::new();
716 assert!(is_excluded_page(Path::new("error500.html"), &meta));
717 }
718
719 #[test]
724 fn test_group_pages_by_section() {
725 let entries = vec![
726 (
727 "Home".to_string(),
728 "/index.html".to_string(),
729 "Welcome".to_string(),
730 ),
731 (
732 "Post 1".to_string(),
733 "/blog/post1.html".to_string(),
734 "First".to_string(),
735 ),
736 (
737 "Post 2".to_string(),
738 "/blog/post2.html".to_string(),
739 "Second".to_string(),
740 ),
741 (
742 "API Ref".to_string(),
743 "/docs/api.html".to_string(),
744 "API docs".to_string(),
745 ),
746 ];
747 let grouped = group_pages_by_section(&entries);
748
749 assert_eq!(grouped.len(), 3);
750 assert!(grouped.contains_key("Pages"));
751 assert!(grouped.contains_key("Blog"));
752 assert!(grouped.contains_key("Docs"));
753 assert_eq!(grouped["Pages"].len(), 1);
754 assert_eq!(grouped["Blog"].len(), 2);
755 assert_eq!(grouped["Docs"].len(), 1);
756 }
757
758 #[test]
759 fn test_group_pages_by_section_root_only() {
760 let entries = vec![
761 (
762 "About".to_string(),
763 "/about.html".to_string(),
764 String::new(),
765 ),
766 (
767 "Contact".to_string(),
768 "/contact.html".to_string(),
769 String::new(),
770 ),
771 ];
772 let grouped = group_pages_by_section(&entries);
773 assert_eq!(grouped.len(), 1);
774 assert_eq!(grouped["Pages"].len(), 2);
775 }
776
777 #[test]
778 fn test_group_pages_by_section_deterministic_order() {
779 let entries = vec![
780 ("Z".to_string(), "/zebra/z.html".to_string(), String::new()),
781 ("A".to_string(), "/alpha/a.html".to_string(), String::new()),
782 ("M".to_string(), "/middle/m.html".to_string(), String::new()),
783 ];
784 let grouped = group_pages_by_section(&entries);
785 let keys: Vec<&String> = grouped.keys().collect();
786 assert_eq!(keys, vec!["Alpha", "Middle", "Zebra"]);
787 }
788
789 #[test]
794 #[serial_test::parallel(ai_failpoint)]
795 fn generate_llms_txt_with_full_config_includes_all_fields() {
796 let dir = tempdir().expect("tempdir");
797 let config = SsgConfig {
798 site_name: "My Site".to_string(),
799 site_description: "A great site".to_string(),
800 base_url: "https://example.com".to_string(),
801 language: "en".to_string(),
802 ..Default::default()
803 };
804
805 generate_llms_txt(dir.path(), Some(&config)).unwrap();
806 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
807 assert!(body.contains("# My Site"));
808 assert!(body.contains("> A great site"));
809 assert!(body.contains("Language: en"));
810 }
811
812 #[test]
813 #[serial_test::parallel(ai_failpoint)]
814 fn generate_llms_txt_without_config_uses_defaults() {
815 let dir = tempdir().expect("tempdir");
816 generate_llms_txt(dir.path(), None).unwrap();
817
818 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
819 assert!(body.contains("# Site"));
820 assert!(body.contains("Language: en"));
821 }
822
823 #[test]
824 #[serial_test::parallel(ai_failpoint)]
825 fn generate_llms_txt_strips_trailing_slash_from_base_url() {
826 let dir = tempdir().expect("tempdir");
827 let config = SsgConfig {
828 site_name: "S".to_string(),
829 site_description: "D".to_string(),
830 base_url: "https://example.com/".to_string(),
831 ..Default::default()
832 };
833
834 write_page(dir.path(), "index.html", "Home", "Welcome", "");
836
837 generate_llms_txt(dir.path(), Some(&config)).unwrap();
838 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
839 assert!(
841 !body.contains("//index.html"),
842 "trailing slash should be normalised:\n{body}"
843 );
844 }
845
846 #[test]
847 #[serial_test::parallel(ai_failpoint)]
848 fn generate_llms_txt_into_missing_parent_returns_err() {
849 let bogus = Path::new("/this/path/should/not/exist");
850 assert!(generate_llms_txt(bogus, None).is_err());
851 }
852
853 #[test]
854 #[serial_test::parallel(ai_failpoint)]
855 fn test_llms_txt_contains_language() {
856 let dir = tempdir().expect("tempdir");
857 let config = SsgConfig {
858 language: "fr".to_string(),
859 ..Default::default()
860 };
861 generate_llms_txt(dir.path(), Some(&config)).unwrap();
862 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
863 assert!(
864 body.contains("Language: fr"),
865 "llms.txt must include Language field:\n{body}"
866 );
867 }
868
869 #[test]
870 #[serial_test::parallel(ai_failpoint)]
871 fn test_llms_txt_contains_language_defaults_to_en() {
872 let dir = tempdir().expect("tempdir");
873 let config = SsgConfig {
874 language: String::new(),
875 ..Default::default()
876 };
877 generate_llms_txt(dir.path(), Some(&config)).unwrap();
878 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
879 assert!(
880 body.contains("Language: en"),
881 "empty language should default to en:\n{body}"
882 );
883 }
884
885 #[test]
886 #[serial_test::parallel(ai_failpoint)]
887 fn test_llms_txt_excludes_drafts() {
888 let dir = tempdir().expect("tempdir");
889 write_page(dir.path(), "published.html", "Published", "Visible", "");
890 write_page(
891 dir.path(),
892 "draft.html",
893 "Draft Post",
894 "Hidden",
895 r#""draft": true"#,
896 );
897
898 generate_llms_txt(dir.path(), None).unwrap();
899 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
900 assert!(
901 body.contains("Published"),
902 "published page must appear:\n{body}"
903 );
904 assert!(
905 !body.contains("Draft Post"),
906 "draft page must be excluded:\n{body}"
907 );
908 }
909
910 #[test]
911 #[serial_test::parallel(ai_failpoint)]
912 fn test_llms_txt_excludes_private() {
913 let dir = tempdir().expect("tempdir");
914 write_page(dir.path(), "public.html", "Public", "Visible", "");
915 write_page(
916 dir.path(),
917 "secret.html",
918 "Secret",
919 "Hidden",
920 r#""private": true"#,
921 );
922
923 generate_llms_txt(dir.path(), None).unwrap();
924 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
925 assert!(
926 !body.contains("Secret"),
927 "private page must be excluded:\n{body}"
928 );
929 }
930
931 #[test]
932 #[serial_test::parallel(ai_failpoint)]
933 fn test_llms_txt_excludes_404() {
934 let dir = tempdir().expect("tempdir");
935 write_page(dir.path(), "index.html", "Home", "Welcome", "");
936 write_page(dir.path(), "404.html", "Not Found", "Error page", "");
937
938 generate_llms_txt(dir.path(), None).unwrap();
939 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
940 assert!(
941 !body.contains("Not Found"),
942 "404 page must be excluded:\n{body}"
943 );
944 }
945
946 #[test]
947 #[serial_test::parallel(ai_failpoint)]
948 fn test_llms_txt_contains_sections() {
949 let dir = tempdir().expect("tempdir");
950 write_page(dir.path(), "index.html", "Home", "Welcome", "");
951 write_page(dir.path(), "blog/post.html", "My Post", "A blog post", "");
952 write_page(
953 dir.path(),
954 "docs/api.html",
955 "API Docs",
956 "API reference",
957 "",
958 );
959
960 generate_llms_txt(dir.path(), None).unwrap();
961 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
962 assert!(
963 body.contains("## Pages"),
964 "should have Pages section:\n{body}"
965 );
966 assert!(
967 body.contains("## Blog"),
968 "should have Blog section:\n{body}"
969 );
970 assert!(
971 body.contains("## Docs"),
972 "should have Docs section:\n{body}"
973 );
974 assert!(
975 body.contains("- [My Post]"),
976 "should contain page link:\n{body}"
977 );
978 }
979
980 #[test]
981 #[serial_test::parallel(ai_failpoint)]
982 fn test_llms_txt_contains_disallow_section() {
983 let dir = tempdir().expect("tempdir");
984 fs::write(
985 dir.path().join("robots.txt"),
986 "User-agent: *\nDisallow: /admin/\n",
987 )
988 .unwrap();
989
990 generate_llms_txt(dir.path(), None).unwrap();
991 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
992 assert!(
993 body.contains("## Disallow"),
994 "should have Disallow section:\n{body}"
995 );
996 assert!(
997 body.contains("- /admin/"),
998 "should contain disallow pattern:\n{body}"
999 );
1000 }
1001
1002 #[test]
1003 #[serial_test::parallel(ai_failpoint)]
1004 fn test_llms_txt_no_disallow_without_robots() {
1005 let dir = tempdir().expect("tempdir");
1006 generate_llms_txt(dir.path(), None).unwrap();
1007 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
1008 assert!(
1009 !body.contains("## Disallow"),
1010 "no robots.txt means no Disallow section:\n{body}"
1011 );
1012 }
1013
1014 #[test]
1019 #[serial_test::parallel(ai_failpoint)]
1020 fn test_llms_full_txt_contains_body_content() {
1021 let dir = tempdir().expect("tempdir");
1022 write_page(dir.path(), "index.html", "Home", "Welcome home", "");
1023
1024 generate_llms_full_txt(dir.path(), None).unwrap();
1025 let body =
1026 fs::read_to_string(dir.path().join("llms-full.txt")).unwrap();
1027 assert!(body.contains("# Site"), "header present:\n{body}");
1028 assert!(body.contains("Language: en"), "language present:\n{body}");
1029 assert!(body.contains("## [Home]"), "page title present:\n{body}");
1030 assert!(body.contains("Welcome home"), "body text present:\n{body}");
1031 }
1032
1033 #[test]
1034 #[serial_test::parallel(ai_failpoint)]
1035 fn test_llms_full_txt_excludes_drafts() {
1036 let dir = tempdir().expect("tempdir");
1037 write_page(dir.path(), "ok.html", "Visible", "Content", "");
1038 write_page(
1039 dir.path(),
1040 "hidden.html",
1041 "Hidden",
1042 "Secret",
1043 r#""draft": true"#,
1044 );
1045
1046 generate_llms_full_txt(dir.path(), None).unwrap();
1047 let body =
1048 fs::read_to_string(dir.path().join("llms-full.txt")).unwrap();
1049 assert!(body.contains("Visible"), "published page present:\n{body}");
1050 assert!(!body.contains("Hidden"), "draft excluded:\n{body}");
1051 }
1052
1053 #[test]
1054 #[serial_test::parallel(ai_failpoint)]
1055 fn test_llms_full_txt_excludes_404() {
1056 let dir = tempdir().expect("tempdir");
1057 write_page(dir.path(), "index.html", "Home", "Welcome", "");
1058 write_page(dir.path(), "404.html", "Not Found", "Error", "");
1059
1060 generate_llms_full_txt(dir.path(), None).unwrap();
1061 let body =
1062 fs::read_to_string(dir.path().join("llms-full.txt")).unwrap();
1063 assert!(!body.contains("Not Found"), "404 excluded:\n{body}");
1064 }
1065
1066 #[test]
1071 fn test_strip_html_tags() {
1072 assert_eq!(strip_html_tags("<p>hello</p>"), "hello");
1073 assert_eq!(strip_html_tags("<div><b>bold</b> text</div>"), "bold text");
1074 assert_eq!(strip_html_tags("no tags"), "no tags");
1075 assert_eq!(strip_html_tags(""), "");
1076 }
1077
1078 #[test]
1079 fn test_extract_body() {
1080 let html =
1081 "<html><head><title>T</title></head><body>Content</body></html>";
1082 assert_eq!(extract_body(html), "Content");
1083 }
1084
1085 #[test]
1086 fn test_extract_body_with_attributes() {
1087 let html = "<html><body class=\"main\">Content</body></html>";
1088 assert_eq!(extract_body(html), "Content");
1089 }
1090
1091 #[test]
1092 fn test_extract_body_no_body_tag() {
1093 let html = "<p>Just a fragment</p>";
1094 assert_eq!(extract_body(html), html);
1095 }
1096
1097 #[test]
1098 fn test_collapse_whitespace() {
1099 assert_eq!(collapse_whitespace(" hello world "), "hello world");
1100 assert_eq!(collapse_whitespace("no extra"), "no extra");
1101 assert_eq!(collapse_whitespace(""), "");
1102 }
1103
1104 #[test]
1109 fn test_titlecase_word() {
1110 assert_eq!(titlecase_word("blog"), "Blog");
1111 assert_eq!(titlecase_word("DOCS"), "Docs");
1112 assert_eq!(titlecase_word(""), "");
1113 assert_eq!(titlecase_word("a"), "A");
1114 }
1115
1116 #[test]
1121 fn after_compile_missing_site_dir_returns_ok_without_writing() {
1122 let dir = tempdir().expect("tempdir");
1123 let missing = dir.path().join("missing");
1124 let ctx =
1125 PluginContext::new(dir.path(), dir.path(), &missing, dir.path());
1126
1127 AiPlugin.after_compile(&ctx).expect("missing site is fine");
1128 assert!(!missing.exists());
1129 assert!(!dir.path().join("llms.txt").exists());
1130 }
1131
1132 #[test]
1133 #[serial_test::parallel(ai_failpoint)]
1134 fn after_compile_injects_max_snippet_meta_tag() {
1135 let (_tmp, site, ctx) = make_site();
1136 let html = "<html><head><title>X</title></head><body></body></html>";
1137 fs::write(site.join("index.html"), html).unwrap();
1138
1139 AiPlugin.after_compile(&ctx).unwrap();
1140 let output = fs::read_to_string(site.join("index.html")).unwrap();
1141 assert!(output.contains("max-snippet"));
1142 assert!(output.contains("max-image-preview:large"));
1143 }
1144
1145 #[test]
1146 #[serial_test::parallel(ai_failpoint)]
1147 fn after_compile_creates_llms_txt_in_site_root() {
1148 let (_tmp, site, ctx) = make_site();
1149 AiPlugin.after_compile(&ctx).unwrap();
1150 assert!(site.join("llms.txt").exists());
1151 }
1152
1153 #[test]
1154 #[serial_test::parallel(ai_failpoint)]
1155 fn after_compile_creates_llms_full_txt_in_site_root() {
1156 let (_tmp, site, ctx) = make_site();
1157 AiPlugin.after_compile(&ctx).unwrap();
1158 assert!(site.join("llms-full.txt").exists());
1159 }
1160
1161 #[test]
1162 #[serial_test::parallel(ai_failpoint)]
1163 fn after_compile_idempotent_does_not_duplicate_meta_tag() {
1164 let (_tmp, site, ctx) = make_site();
1165 let html = "<html><head><title>X</title></head><body></body></html>";
1166 fs::write(site.join("index.html"), html).unwrap();
1167
1168 AiPlugin.after_compile(&ctx).unwrap();
1169 AiPlugin.after_compile(&ctx).unwrap();
1170
1171 let output = fs::read_to_string(site.join("index.html")).unwrap();
1172 assert_eq!(output.matches("max-snippet").count(), 1);
1173 }
1174
1175 #[test]
1176 #[serial_test::parallel(ai_failpoint)]
1177 fn after_compile_skips_html_files_without_head_tag() {
1178 let (_tmp, site, ctx) = make_site();
1179 fs::write(site.join("fragment.html"), "<p>just a fragment</p>")
1180 .unwrap();
1181
1182 AiPlugin.after_compile(&ctx).unwrap();
1183 let output = fs::read_to_string(site.join("fragment.html")).unwrap();
1184 assert!(!output.contains("max-snippet"));
1185 assert_eq!(output, "<p>just a fragment</p>");
1186 }
1187
1188 #[test]
1189 #[serial_test::parallel(ai_failpoint)]
1190 fn after_compile_processes_files_in_subdirectories() {
1191 let (_tmp, site, ctx) = make_site();
1192 let nested = site.join("blog");
1193 fs::create_dir_all(&nested).unwrap();
1194 fs::write(
1195 nested.join("post.html"),
1196 "<html><head></head><body></body></html>",
1197 )
1198 .unwrap();
1199
1200 AiPlugin.after_compile(&ctx).unwrap();
1201 let output = fs::read_to_string(nested.join("post.html")).unwrap();
1202 assert!(output.contains("max-snippet"));
1203 }
1204
1205 #[test]
1206 #[serial_test::parallel(ai_failpoint)]
1207 fn after_compile_logs_warning_for_pages_with_missing_alt() {
1208 let (_tmp, site, ctx) = make_site();
1209 fs::write(
1210 site.join("bad.html"),
1211 r#"<html><head></head><body><img src="a.jpg"></body></html>"#,
1212 )
1213 .unwrap();
1214 fs::write(
1215 site.join("worse.html"),
1216 r#"<html><head></head><body><img src="a.jpg" alt=""></body></html>"#,
1217 )
1218 .unwrap();
1219
1220 AiPlugin.after_compile(&ctx).unwrap();
1221 let bad = fs::read_to_string(site.join("bad.html")).unwrap();
1222 assert!(bad.contains("max-snippet"));
1223 }
1224
1225 #[test]
1226 #[serial_test::parallel(ai_failpoint)]
1227 fn after_compile_does_not_rewrite_unchanged_files() {
1228 let (_tmp, site, ctx) = make_site();
1229 let html = "<html><head><meta name=\"robots\" content=\"max-snippet:-1\"></head><body></body></html>";
1230 fs::write(site.join("index.html"), html).unwrap();
1231 let original_mtime = fs::metadata(site.join("index.html"))
1232 .unwrap()
1233 .modified()
1234 .unwrap();
1235
1236 AiPlugin.after_compile(&ctx).unwrap();
1237 let after = fs::read_to_string(site.join("index.html")).unwrap();
1238 assert_eq!(after, html, "unchanged file body must be preserved");
1239 let _ = original_mtime;
1240 }
1241
1242 #[test]
1247 #[serial_test::parallel(ai_failpoint)]
1248 fn collect_html_files_returns_empty_for_missing_directory() {
1249 let dir = tempdir().expect("tempdir");
1250 let result = collect_html_files(&dir.path().join("missing")).unwrap();
1251 assert!(result.is_empty());
1252 }
1253
1254 #[test]
1255 #[serial_test::parallel(ai_failpoint)]
1256 fn collect_html_files_filters_non_html_extensions() {
1257 let dir = tempdir().expect("tempdir");
1258 fs::write(dir.path().join("a.html"), "").unwrap();
1259 fs::write(dir.path().join("b.css"), "").unwrap();
1260 fs::write(dir.path().join("c.js"), "").unwrap();
1261
1262 let result = collect_html_files(dir.path()).unwrap();
1263 assert_eq!(result.len(), 1);
1264 }
1265
1266 #[test]
1267 #[serial_test::parallel(ai_failpoint)]
1268 fn collect_html_files_recurses_into_nested_subdirectories() {
1269 let dir = tempdir().expect("tempdir");
1270 let nested = dir.path().join("a").join("b");
1271 fs::create_dir_all(&nested).unwrap();
1272 fs::write(dir.path().join("top.html"), "").unwrap();
1273 fs::write(nested.join("deep.html"), "").unwrap();
1274
1275 let result = collect_html_files(dir.path()).unwrap();
1276 assert_eq!(result.len(), 2);
1277 }
1278
1279 #[test]
1280 #[serial_test::parallel(ai_failpoint)]
1281 fn collect_html_files_returns_results_sorted() {
1282 let dir = tempdir().expect("tempdir");
1283 for name in ["zebra.html", "apple.html", "mango.html"] {
1284 fs::write(dir.path().join(name), "").unwrap();
1285 }
1286 let result = collect_html_files(dir.path()).unwrap();
1287 let names: Vec<_> = result
1288 .iter()
1289 .map(|p| p.file_name().unwrap().to_str().unwrap())
1290 .collect();
1291 assert_eq!(names, vec!["apple.html", "mango.html", "zebra.html"]);
1292 }
1293
1294 #[test]
1299 fn test_is_excluded_page_draft_false_is_not_excluded() {
1300 let mut meta = serde_json::Map::new();
1301 let _ =
1302 meta.insert("draft".to_string(), serde_json::Value::Bool(false));
1303 assert!(!is_excluded_page(Path::new("post.html"), &meta));
1304 }
1305
1306 #[test]
1307 fn test_is_excluded_page_private_string_true() {
1308 let mut meta = serde_json::Map::new();
1309 let _ = meta.insert(
1310 "private".to_string(),
1311 serde_json::Value::String("true".to_string()),
1312 );
1313 assert!(is_excluded_page(Path::new("post.html"), &meta));
1314 }
1315
1316 #[test]
1317 fn test_is_excluded_page_private_false_is_not_excluded() {
1318 let mut meta = serde_json::Map::new();
1319 let _ =
1320 meta.insert("private".to_string(), serde_json::Value::Bool(false));
1321 assert!(!is_excluded_page(Path::new("post.html"), &meta));
1322 }
1323
1324 #[test]
1330 #[serial_test::parallel(ai_failpoint)]
1331 fn collect_page_entries_unreadable_sidecar_falls_back_to_empty_meta() {
1332 let dir = tempdir().expect("tempdir");
1333 fs::write(
1334 dir.path().join("page.html"),
1335 "<html><head></head><body>Hi</body></html>",
1336 )
1337 .unwrap();
1338 fs::create_dir_all(dir.path().join("page.meta.json")).unwrap();
1342
1343 let entries = collect_page_entries(dir.path()).unwrap();
1344 assert!(
1345 entries.is_empty(),
1346 "titleless page must be dropped: {entries:?}"
1347 );
1348 }
1349
1350 #[test]
1351 #[serial_test::parallel(ai_failpoint)]
1352 fn llms_full_txt_unreadable_sidecar_skips_page() {
1353 let dir = tempdir().expect("tempdir");
1354 fs::write(
1355 dir.path().join("page.html"),
1356 "<html><head></head><body>Hidden body</body></html>",
1357 )
1358 .unwrap();
1359 fs::create_dir_all(dir.path().join("page.meta.json")).unwrap();
1360
1361 generate_llms_full_txt(dir.path(), None).unwrap();
1362 let body =
1363 fs::read_to_string(dir.path().join("llms-full.txt")).unwrap();
1364 assert!(
1365 !body.contains("Hidden body"),
1366 "page without readable sidecar (no title) must be skipped:\n{body}"
1367 );
1368 }
1369
1370 #[test]
1375 #[serial_test::parallel(ai_failpoint)]
1376 fn llms_txt_entry_without_description_omits_colon_suffix() {
1377 let dir = tempdir().expect("tempdir");
1378 write_page(dir.path(), "index.html", "Home", "", "");
1379
1380 generate_llms_txt(dir.path(), None).unwrap();
1381 let body = fs::read_to_string(dir.path().join("llms.txt")).unwrap();
1382 assert!(
1383 body.contains("- [Home](/index.html)\n"),
1384 "description-less entry must be a bare link:\n{body}"
1385 );
1386 assert!(
1387 !body.contains("- [Home](/index.html):"),
1388 "no trailing colon without a description:\n{body}"
1389 );
1390 }
1391
1392 #[test]
1397 #[serial_test::parallel(ai_failpoint)]
1398 fn llms_full_txt_uses_canonical_root_for_urls() {
1399 let dir = tempdir().expect("tempdir");
1400 write_page(dir.path(), "index.html", "Home", "Welcome", "");
1401
1402 let config = SsgConfig {
1403 site_name: "S".to_string(),
1404 base_url: "https://example.com/".to_string(),
1405 ..Default::default()
1406 };
1407 generate_llms_full_txt(dir.path(), Some(&config)).unwrap();
1408 let body =
1409 fs::read_to_string(dir.path().join("llms-full.txt")).unwrap();
1410 assert!(
1411 body.contains("## [Home](https://example.com/index.html)"),
1412 "page URL must be prefixed with the canonical root:\n{body}"
1413 );
1414 }
1415
1416 #[test]
1426 fn test_is_excluded_page_draft_string_non_true_is_not_excluded() {
1427 let mut meta = serde_json::Map::new();
1428 let _ = meta.insert(
1429 "draft".to_string(),
1430 serde_json::Value::String("no".to_string()),
1431 );
1432 assert!(!is_excluded_page(Path::new("post.html"), &meta));
1433 }
1434
1435 #[test]
1436 fn test_is_excluded_page_private_string_non_true_is_not_excluded() {
1437 let mut meta = serde_json::Map::new();
1438 let _ = meta.insert(
1439 "private".to_string(),
1440 serde_json::Value::String("no".to_string()),
1441 );
1442 assert!(!is_excluded_page(Path::new("post.html"), &meta));
1443 }
1444
1445 #[test]
1453 #[serial_test::parallel(ai_failpoint)]
1454 fn after_compile_propagates_llms_txt_write_error() {
1455 let (_tmp, site, ctx) = make_site();
1456 fs::create_dir_all(site.join("llms.txt")).unwrap();
1460 let err = AiPlugin.after_compile(&ctx).unwrap_err();
1461 assert!(!format!("{err}").is_empty());
1462 }
1463
1464 #[test]
1465 #[serial_test::parallel(ai_failpoint)]
1466 fn after_compile_propagates_llms_full_txt_write_error() {
1467 let (_tmp, site, ctx) = make_site();
1468 fs::create_dir_all(site.join("llms-full.txt")).unwrap();
1471 let err = AiPlugin.after_compile(&ctx).unwrap_err();
1472 assert!(!format!("{err}").is_empty());
1473 }
1474
1475 #[test]
1476 #[serial_test::parallel(ai_failpoint)]
1477 #[cfg(unix)]
1478 fn after_compile_fails_when_html_file_is_unreadable() {
1479 use std::os::unix::fs::PermissionsExt;
1480 let (_tmp, site, ctx) = make_site();
1481 let html = site.join("locked.html");
1482 fs::write(&html, "<html><head></head><body></body></html>").unwrap();
1483 fs::set_permissions(&html, fs::Permissions::from_mode(0o000)).unwrap();
1484
1485 let res = AiPlugin.after_compile(&ctx);
1486
1487 let _ = fs::set_permissions(&html, fs::Permissions::from_mode(0o644));
1488 if let Err(e) = res {
1491 assert!(!format!("{e}").is_empty());
1492 }
1493 }
1494}
1495
1496#[cfg(all(test, feature = "test-fault-injection"))]
1505mod fault_tests {
1506 use super::*;
1507 use crate::plugin::PluginContext;
1508 use tempfile::tempdir;
1509
1510 struct FailGuard(&'static str);
1512
1513 impl Drop for FailGuard {
1514 fn drop(&mut self) {
1515 let _ = fail::cfg(self.0, "off");
1516 }
1517 }
1518
1519 #[test]
1520 #[serial_test::serial(ai_failpoint)]
1521 fn collect_html_files_third_call_failpoint_propagates() {
1522 let _guard = FailGuard("ai::collect-html-files");
1523 fail::cfg("ai::collect-html-files", "2*off->1*return")
1527 .expect("activate failpoint");
1528
1529 let dir = tempdir().unwrap();
1530 let site = dir.path().join("site");
1531 fs::create_dir_all(&site).unwrap();
1532 let ctx = PluginContext::new(dir.path(), dir.path(), &site, dir.path());
1533
1534 let err = AiPlugin
1535 .after_compile(&ctx)
1536 .expect_err("third collect_html_files call must propagate");
1537 assert!(format!("{err:?}").contains("injected: ai::collect-html-files"));
1538 }
1539}