1use crate::error::SsgError;
7use crate::util::head_dom::extract_head_meta;
8use std::path::{Path, PathBuf};
9
10pub fn extract_title(html: &str) -> String {
26 extract_head_meta(html).title
27}
28
29pub(super) fn extract_description(html: &str, max_len: usize) -> String {
35 let content = extract_main_content(html);
36
37 let clean = strip_inline_tags(&content, &["script", "style"]);
38
39 let text = strip_tags(&clean);
40 let trimmed = text.trim();
41 truncate_at_word_boundary(trimmed, max_len)
42}
43
44fn extract_main_content(html: &str) -> String {
47 if let Some(inner) = extract_tag_inner(html, "main") {
48 return inner;
49 }
50
51 let body =
52 extract_tag_inner(html, "body").unwrap_or_else(|| html.to_string());
53 strip_inline_tags(&body, &["script", "style", "nav", "header", "footer"])
54}
55
56fn extract_tag_inner(html: &str, tag_name: &str) -> Option<String> {
58 let open = format!("<{tag_name}");
59 let close = format!("</{tag_name}>");
60 let start = html.find(&open)?;
61 let after = &html[start..];
62 let gt = after.find('>')?;
63 let inner = &after[gt + 1..];
64 if let Some(end) = inner.find(&close) {
65 Some(inner[..end].to_string())
66 } else {
67 Some(inner.to_string())
68 }
69}
70
71fn strip_inline_tags(html: &str, tags: &[&str]) -> String {
73 let mut clean = html.to_string();
74 for tag in tags {
75 let open = format!("<{tag}");
76 let close = format!("</{tag}>");
77 while let Some(start) = clean.find(&open) {
78 if let Some(end) = clean[start..].find(&close) {
79 clean.replace_range(start..start + end + close.len(), " ");
80 } else {
81 break;
82 }
83 }
84 }
85 clean
86}
87
88fn truncate_at_word_boundary(text: &str, max_len: usize) -> String {
90 if text.len() <= max_len {
91 return text.to_string();
92 }
93 let mut end = max_len;
94 while end > 0 && !text.is_char_boundary(end) {
95 end -= 1;
96 }
97 let truncated = &text[..end];
98 if let Some(last_space) = truncated.rfind(' ') {
99 truncated[..last_space].to_string()
100 } else {
101 truncated.to_string()
102 }
103}
104
105pub(super) fn strip_tags(html: &str) -> String {
107 let mut result = String::with_capacity(html.len());
108 let mut in_tag = false;
109 for ch in html.chars() {
110 match ch {
111 '<' => in_tag = true,
112 '>' => {
113 in_tag = false;
114 result.push(' ');
115 }
116 _ if !in_tag => result.push(ch),
117 _ => {}
118 }
119 }
120 let mut collapsed = String::with_capacity(result.len());
122 let mut prev_space = false;
123 for ch in result.chars() {
124 if ch.is_whitespace() {
125 if !prev_space {
126 collapsed.push(' ');
127 prev_space = true;
128 }
129 } else {
130 collapsed.push(ch);
131 prev_space = false;
132 }
133 }
134 collapsed.trim().to_string()
135}
136
137#[allow(dead_code)] pub(super) fn collect_html_files(dir: &Path) -> Result<Vec<PathBuf>, SsgError> {
140 crate::walk::walk_files(dir, "html")
141}
142
143const MAX_ENTITY_BYTES: usize = 32;
146
147fn scan_existing_entity(bytes: &[u8], start: usize) -> Option<usize> {
153 let limit = bytes.len().min(start + MAX_ENTITY_BYTES);
154 let mut i = start + 1;
155 if i >= limit {
156 return None;
157 }
158 let numeric = bytes[i] == b'#';
159 if numeric {
160 i += 1;
161 if i < limit && (bytes[i] == b'x' || bytes[i] == b'X') {
162 i += 1;
163 }
164 }
165 let value_start = i;
166 while i < limit {
167 match bytes[i] {
168 b';' if i > value_start => return Some(i + 1),
169 b'0'..=b'9' => i += 1,
170 b'a'..=b'z' | b'A'..=b'Z' if !numeric => i += 1,
171 b'a'..=b'f' | b'A'..=b'F' if numeric => i += 1,
172 _ => return None,
173 }
174 }
175 None
176}
177
178pub(super) fn escape_attr(s: &str) -> String {
194 let bytes = s.as_bytes();
195 let mut out = String::with_capacity(s.len());
196 let mut i = 0;
197 let mut start = 0;
198 while i < bytes.len() {
199 let replacement = match bytes[i] {
200 b'&' => {
201 if let Some(end) = scan_existing_entity(bytes, i) {
202 i = end;
204 continue;
205 }
206 "&"
207 }
208 b'"' => """,
209 b'<' => "<",
210 b'>' => ">",
211 _ => {
212 i += 1;
213 continue;
214 }
215 };
216 if start < i {
217 out.push_str(&s[start..i]);
218 }
219 out.push_str(replacement);
220 i += 1;
221 start = i;
222 }
223 if start < s.len() {
224 out.push_str(&s[start..]);
225 }
226 out
227}
228
229pub fn has_meta_tag(html: &str, attr: &str) -> bool {
248 html.contains(&format!("<meta property=\"{attr}\""))
249 || html.contains(&format!("<meta property='{attr}'"))
250 || html.contains(&format!("<meta name=\"{attr}\""))
251 || html.contains(&format!("<meta name='{attr}'"))
252}
253
254pub(super) fn extract_canonical(html: &str) -> String {
260 extract_head_meta(html).canonical
261}
262
263pub(super) fn extract_existing_meta(html: &str, attr: &str) -> String {
270 use std::cell::RefCell;
271 use std::rc::Rc;
272
273 use lol_html::element;
274
275 use crate::util::html_rewriter::rewrite_html;
276
277 let found: Rc<RefCell<String>> = Rc::new(RefCell::new(String::new()));
284 let sink = Rc::clone(&found);
285 let want = attr.to_ascii_lowercase();
286
287 let handlers = vec![element!("meta", move |el| {
288 if !sink.borrow().is_empty() {
289 return Ok(());
290 }
291 let keyed = ["name", "property"].iter().any(|k| {
294 el.get_attribute(k)
295 .is_some_and(|v| v.eq_ignore_ascii_case(&want))
296 });
297 if keyed {
298 if let Some(content) = el.get_attribute("content") {
299 let value = content.trim();
300 if !value.is_empty() {
301 sink.borrow_mut().push_str(value);
302 }
303 }
304 }
305 Ok(())
306 })];
307
308 let _ = rewrite_html(html, handlers);
309
310 let out = found.borrow().clone();
311 out
312}
313
314pub(super) fn extract_html_lang(html: &str) -> String {
319 extract_head_meta(html).lang
320}
321
322pub(super) fn extract_first_content_image(html: &str) -> String {
324 let search_region = if let Some(start) = html.find("<main") {
326 &html[start..]
327 } else if let Some(start) = html.find("<article") {
328 &html[start..]
329 } else {
330 return String::new();
331 };
332
333 if let Some(img_pos) = search_region.find("<img") {
334 let after_img = &search_region[img_pos..];
335 let tag_end = after_img.find('>').unwrap_or(500).min(500);
336 let img_tag = &after_img[..tag_end];
337 if let Some(src_pos) = img_tag.find("src=\"") {
338 let after_src = &img_tag[src_pos + 5..];
339 if let Some(end) = after_src.find('"') {
340 return after_src[..end].to_string();
341 }
342 }
343 }
344 String::new()
345}
346
347pub(super) fn extract_meta_author(html: &str) -> String {
349 let from_meta = extract_existing_meta(html, "author");
351 if !from_meta.is_empty() {
352 return from_meta;
353 }
354 for pattern in &["class=\"author\">", "class='author'>", "rel=\"author\">"]
356 {
357 if let Some(pos) = html.find(pattern) {
358 let after = &html[pos + pattern.len()..];
359 if let Some(end) = after.find('<') {
360 let name = after[..end].trim();
361 let name = name.strip_prefix("by ").unwrap_or(name).trim();
363 if !name.is_empty() {
364 return name.to_string();
365 }
366 }
367 }
368 }
369 String::new()
370}
371
372pub(super) fn extract_date_from_html(
374 html: &str,
375 field: &str,
376) -> Option<String> {
377 let pattern = format!("\"{field}\":\"");
378 if let Some(pos) = html.find(&pattern) {
379 let after = &html[pos + pattern.len()..];
380 if let Some(end) = after.find('"') {
381 let date = &after[..end];
382 if !date.is_empty() {
383 return Some(date.to_string());
384 }
385 }
386 }
387 None
388}
389
390pub(super) fn extract_meta_date(html: &str) -> Option<String> {
392 let meta = extract_existing_meta(html, "article:published_time");
394 if !meta.is_empty() {
395 return Some(meta);
396 }
397 if let Some(pos) = html.find("datetime=\"") {
399 let after = &html[pos + 10..];
400 if let Some(end) = after.find('"') {
401 let date = &after[..end];
402 if !date.is_empty() {
403 return Some(date.to_string());
404 }
405 }
406 }
407 None
408}
409
410#[allow(dead_code)] pub(super) fn collect_html_files_recursive(
413 dir: &Path,
414) -> Result<Vec<PathBuf>, SsgError> {
415 crate::walk::walk_files(dir, "html")
416}
417
418#[cfg(test)]
419mod tests {
420
421 #[test]
425 fn ac1_title_ignores_commented_title() {
426 let html = "<html><head><!-- <title>Old</title> --><title>Real</title></head><body></body></html>";
427 assert_eq!(extract_title(html), "Real");
428 }
429
430 #[test]
432 fn ac3_canonical_is_detected() {
433 let html = r#"<html><head><link rel="canonical" href="https://x"></head><body></body></html>"#;
434 assert_eq!(extract_canonical(html), "https://x");
435 }
436
437 #[test]
439 fn ac_meta_ignores_commented_meta() {
440 let html = concat!(
441 "<html><head>",
442 "<!-- <meta name=\"twitter:image\" content=\"COMMENTED\"> -->",
443 "<meta name=\"twitter:image\" content=\"REAL\">",
444 "</head><body></body></html>"
445 );
446 assert_eq!(extract_existing_meta(html, "twitter:image"), "REAL");
447 }
448
449 #[test]
451 fn ac_meta_ignores_meta_in_pre_block() {
452 let html = concat!(
453 "<html><head><meta name=\"description\" content=\"REAL\"></head>",
454 "<body><pre><meta name=\"description\" content=\"EXAMPLE\"></pre></body></html>"
455 );
456 assert_eq!(extract_existing_meta(html, "description"), "REAL");
457 }
458
459 use super::*;
460 use std::fs;
461 use tempfile::tempdir;
462
463 #[test]
464 fn extract_title_from_html() {
465 let html = "<html><head><title>Test Page</title></head></html>";
466 assert_eq!(extract_title(html), "Test Page");
467 }
468
469 #[test]
470 fn extract_existing_meta_minified_unquoted_and_reordered() {
471 let html = "<head><meta content=https://ex.test/img.png \
474 name=twitter:image></head>";
475 assert_eq!(
476 extract_existing_meta(html, "twitter:image"),
477 "https://ex.test/img.png"
478 );
479 let html2 =
480 "<head><meta content=\"https://ex.test/og.png\" property=og:image></head>";
481 assert_eq!(
482 extract_existing_meta(html2, "og:image"),
483 "https://ex.test/og.png"
484 );
485 }
486
487 #[test]
488 fn extract_existing_meta_quoted_forms_still_work() {
489 let html = r#"<meta name="author" content="Alice">"#;
490 assert_eq!(extract_existing_meta(html, "author"), "Alice");
491 let html2 = r#"<meta property='og:image' content='/x.png'>"#;
492 assert_eq!(extract_existing_meta(html2, "og:image"), "/x.png");
493 }
494
495 #[test]
496 fn extract_existing_meta_absent_returns_empty() {
497 let html = "<head><meta name=viewport content=width=device-width>\
498 </head>";
499 assert_eq!(extract_existing_meta(html, "og:image"), "");
500 }
501
502 #[test]
503 fn extract_existing_meta_skips_empty_content_and_keeps_scanning() {
504 let html = "<meta name=author content=\"\">\
505 <meta name=author content=\"Bea\">";
506 assert_eq!(extract_existing_meta(html, "author"), "Bea");
507 }
508
509 #[test]
510 fn extract_title_empty_no_tag() {
511 let html = "<html><head></head><body>Hello</body></html>";
512 assert_eq!(extract_title(html), "");
513 }
514
515 #[test]
516 fn extract_title_empty_tag() {
517 let html = "<html><head><title></title></head></html>";
518 assert_eq!(extract_title(html), "");
519 }
520
521 #[test]
522 fn extract_title_nested_tags() {
523 let html = "<title><span>Inner</span></title>";
524 assert_eq!(extract_title(html), "Inner");
526 }
527
528 #[test]
529 fn extract_description_from_body() {
530 let html = "<html><body><main><p>Short description here.</p></main></body></html>";
531 let desc = extract_description(html, 200);
532 assert!(desc.contains("Short description here"));
533 }
534
535 #[test]
536 fn extract_description_truncation() {
537 let long_text = "word ".repeat(100);
538 let html = format!("<main><p>{long_text}</p></main>");
539 let desc = extract_description(&html, 50);
540 assert!(desc.len() <= 50);
541 }
542
543 #[test]
544 fn strip_tags_basic() {
545 assert_eq!(strip_tags("<p>Hello <b>world</b></p>"), "Hello world");
546 }
547
548 #[test]
549 fn strip_tags_empty() {
550 assert_eq!(strip_tags(""), "");
551 }
552
553 #[test]
554 fn strip_tags_no_tags() {
555 assert_eq!(strip_tags("plain text"), "plain text");
556 }
557
558 #[test]
559 fn strip_tags_self_closing() {
560 let result = strip_tags("<img src=\"x\"/>text");
561 assert!(result.contains("text"));
562 assert!(!result.contains("img"));
563 }
564
565 #[test]
566 fn truncate_short_text_unchanged() {
567 assert_eq!(truncate_at_word_boundary("short", 100), "short");
568 }
569
570 #[test]
571 fn truncate_long_text_at_word() {
572 let text = "one two three four five six";
573 let result = truncate_at_word_boundary(text, 15);
574 assert!(result.len() <= 15);
575 assert!(!result.ends_with(' '));
577 assert_eq!(result, "one two three");
578 }
579
580 #[test]
581 fn truncate_unicode() {
582 let text = "日本語 テスト データ";
583 let result = truncate_at_word_boundary(text, 15);
584 assert!(result.len() <= 15);
586 }
587
588 #[test]
589 fn collect_html_files_finds_files() {
590 let tmp = tempdir().unwrap();
591 let sub = tmp.path().join("sub");
592 fs::create_dir_all(&sub).unwrap();
593 fs::write(tmp.path().join("index.html"), "<html></html>").unwrap();
594 fs::write(sub.join("page.html"), "<html></html>").unwrap();
595
596 let files = collect_html_files(tmp.path()).unwrap();
597 assert_eq!(files.len(), 2);
598 }
599
600 #[test]
601 fn collect_html_files_recursive_finds_files() {
602 let tmp = tempdir().unwrap();
603 let sub = tmp.path().join("sub");
604 fs::create_dir_all(&sub).unwrap();
605 fs::write(tmp.path().join("index.html"), "<html></html>").unwrap();
606 fs::write(sub.join("page.html"), "<html></html>").unwrap();
607 fs::write(sub.join("style.css"), "body{}").unwrap();
608
609 let files = collect_html_files_recursive(tmp.path()).unwrap();
610 assert_eq!(files.len(), 2);
611 assert!(files.iter().all(|p| p.extension().unwrap() == "html"));
612 }
613
614 #[test]
615 fn collect_html_files_recursive_empty_dir() {
616 let tmp = tempdir().unwrap();
617 let files = collect_html_files_recursive(tmp.path()).unwrap();
618 assert!(files.is_empty());
619 }
620
621 #[test]
622 fn escape_attr_special_chars() {
623 assert_eq!(escape_attr("a&b<c>d\"e"), "a&b<c>d"e");
624 }
625
626 #[test]
630 fn escape_attr_preserves_existing_entities() {
631 assert_eq!(escape_attr("A & B"), "A & B");
632 assert_eq!(escape_attr("A < B"), "A < B");
633 assert_eq!(escape_attr("A ' B"), "A ' B");
634 assert_eq!(escape_attr("A ' B"), "A ' B");
635 assert_eq!(escape_attr(" "), " ");
636 }
637
638 #[test]
641 fn escape_attr_is_idempotent() {
642 for input in [
643 "AI, Payments & Post-Quantum Cryptography",
644 "a&b<c>d\"e",
645 "Tom & Jerry's <b>show</b>",
646 "already & escaped",
647 "mixed & already & both",
648 "",
649 "no metacharacters at all",
650 ] {
651 let once = escape_attr(input);
652 let twice = escape_attr(&once);
653 assert_eq!(once, twice, "not idempotent for {input:?}");
654 }
655 }
656
657 #[test]
661 fn escape_attr_still_escapes_bare_ampersands() {
662 assert_eq!(escape_attr("Tom & Jerry"), "Tom & Jerry");
663 assert_eq!(escape_attr("a & b & c"), "a & b & c");
664 assert_eq!(escape_attr("&"), "&amp");
666 assert_eq!(escape_attr("&;"), "&;");
667 assert_eq!(escape_attr("&#;"), "&#;");
668 let long = format!("&{};", "a".repeat(40));
670 assert!(escape_attr(&long).starts_with("&"));
671 }
672
673 #[test]
674 fn has_meta_tag_present() {
675 let html = r#"<meta property="og:title" content="Hi">"#;
676 assert!(has_meta_tag(html, "og:title"));
677 }
678
679 #[test]
680 fn has_meta_tag_absent() {
681 let html = "<html><head></head></html>";
682 assert!(!has_meta_tag(html, "og:title"));
683 }
684
685 #[test]
686 fn extract_canonical_found() {
687 let html = r#"<link rel="canonical" href="https://example.com/page">"#;
688 assert_eq!(extract_canonical(html), "https://example.com/page");
689 }
690
691 #[test]
692 fn extract_canonical_missing() {
693 let html = "<html><head></head></html>";
694 assert_eq!(extract_canonical(html), "");
695 }
696
697 #[test]
698 fn extract_existing_meta_by_name() {
699 let html = r#"<meta name="author" content="Alice">"#;
700 assert_eq!(extract_existing_meta(html, "author"), "Alice");
701 }
702
703 #[test]
704 fn extract_html_lang_found() {
705 let html = r#"<html lang="fr"><head></head></html>"#;
706 assert_eq!(extract_html_lang(html), "fr");
707 }
708
709 #[test]
710 fn extract_html_lang_missing() {
711 let html = "<html><head></head></html>";
712 assert_eq!(extract_html_lang(html), "");
713 }
714
715 #[test]
716 fn extract_date_from_html_found() {
717 let html = r#"{"datePublished":"2025-01-15"}"#;
718 assert_eq!(
719 extract_date_from_html(html, "datePublished"),
720 Some("2025-01-15".to_string())
721 );
722 }
723
724 #[test]
725 fn extract_date_from_html_missing() {
726 assert_eq!(
727 extract_date_from_html("<html></html>", "datePublished"),
728 None
729 );
730 }
731
732 #[test]
733 fn extract_existing_meta_skips_tag_without_content_attribute() {
734 let html = "<meta name=\"author\">\
737 <meta name=\"author\" content=\"Cid\">";
738 assert_eq!(extract_existing_meta(html, "author"), "Cid");
739 }
740
741 #[test]
742 fn extract_existing_meta_no_content_anywhere_returns_empty() {
743 let html = "<meta name=\"author\">";
744 assert_eq!(extract_existing_meta(html, "author"), "");
745 }
746
747 #[test]
748 fn extract_first_content_image_src_without_closing_quote() {
749 let html = "<main><img src=\"broken.png</main>";
752 assert_eq!(extract_first_content_image(html), "");
753 }
754
755 #[test]
756 fn extract_first_content_image_img_without_src_attribute() {
757 let html = "<main><img alt=\"decorative\"></main>";
758 assert_eq!(extract_first_content_image(html), "");
759 }
760
761 #[test]
762 fn extract_meta_author_byline_with_empty_name_returns_empty() {
763 let html = "<html><body><span class=\"author\"></span></body></html>";
766 assert_eq!(extract_meta_author(html), "");
767 }
768
769 #[test]
770 fn extract_meta_author_byline_without_following_tag_returns_empty() {
771 let html = "<span class=\"author\">Dana";
774 assert_eq!(extract_meta_author(html), "");
775 }
776
777 #[test]
778 fn extract_date_from_html_empty_date_returns_none() {
779 let html = r#"{"datePublished":""}"#;
780 assert_eq!(extract_date_from_html(html, "datePublished"), None);
781 }
782
783 #[test]
784 fn extract_date_from_html_unterminated_value_returns_none() {
785 let html = r#"{"datePublished":"2026-01-01"#;
787 assert_eq!(extract_date_from_html(html, "datePublished"), None);
788 }
789
790 #[test]
791 fn extract_meta_date_empty_datetime_returns_none() {
792 let html = r#"<time datetime="">x</time>"#;
793 assert_eq!(extract_meta_date(html), None);
794 }
795
796 #[test]
797 fn extract_meta_date_unterminated_datetime_returns_none() {
798 let html = r#"<time datetime="2026-01-01"#;
799 assert_eq!(extract_meta_date(html), None);
800 }
801}