package main import ( "bytes" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "golang.org/x/net/html" "golang.org/x/net/html/atom" ) var httpClient = &http.Client{Timeout: 30 * time.Second} // fetchMarkdown fetches a URL and returns its main content as Markdown. // On failure it tries the Wayback Machine as a fallback. func fetchMarkdown(rawURL string) (string, error) { md, err := fetchMarkdownDirect(rawURL) if err == nil { return md, nil } // Fallback: Wayback Machine. md, waybackErr := fetchMarkdownWayback(rawURL) if waybackErr == nil { return md + "\n\n---\n*Retrieved from the Wayback Machine.*\n", nil } return "", fmt.Errorf("%v (wayback: %v)", err, waybackErr) } // fetchMarkdownDirect performs a plain HTTP GET and converts HTML to Markdown. func fetchMarkdownDirect(rawURL string) (string, error) { req, err := http.NewRequest("GET", rawURL, nil) if err != nil { return "", err } req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.5") resp, err := httpClient.Do(req) if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP %d", resp.StatusCode) } return parseHTMLToMarkdown(resp.Body) } // fetchMarkdownWayback looks up the most recent Wayback Machine snapshot // for rawURL and fetches its content as Markdown. func fetchMarkdownWayback(rawURL string) (string, error) { apiURL := "https://archive.org/wayback/available?url=" + url.QueryEscape(rawURL) resp, err := httpClient.Get(apiURL) if err != nil { return "", fmt.Errorf("wayback API: %w", err) } defer resp.Body.Close() var result struct { ArchivedSnapshots struct { Closest struct { Available bool `json:"available"` URL string `json:"url"` Status string `json:"status"` } `json:"closest"` } `json:"archived_snapshots"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return "", fmt.Errorf("wayback parse: %w", err) } snap := result.ArchivedSnapshots.Closest if !snap.Available || snap.URL == "" { return "", fmt.Errorf("no snapshot available") } if snap.Status != "200" { return "", fmt.Errorf("snapshot status %s", snap.Status) } wresp, err := httpClient.Get(snap.URL) if err != nil { return "", fmt.Errorf("wayback fetch: %w", err) } defer wresp.Body.Close() if wresp.StatusCode != http.StatusOK { return "", fmt.Errorf("wayback HTTP %d", wresp.StatusCode) } return parseHTMLToMarkdown(wresp.Body) } // parseHTMLToMarkdown reads HTML from r and returns Markdown. func parseHTMLToMarkdown(r io.Reader) (string, error) { body, err := io.ReadAll(io.LimitReader(r, 4<<20)) // 4 MiB if err != nil { return "", err } doc, err := html.Parse(bytes.NewReader(body)) if err != nil { return "", fmt.Errorf("HTML parse: %w", err) } // Prefer
or
; fall back to . root := findNode(doc, atom.Article) if root == nil { root = findNode(doc, atom.Main) } if root == nil { root = findNode(doc, atom.Body) } if root == nil { root = doc } var sb strings.Builder renderMarkdown(&sb, root, 0) return cleanupMarkdown(sb.String()), nil } // --------------------------------------------------------------------------- // HTML → Markdown walker // --------------------------------------------------------------------------- func renderMarkdown(sb *strings.Builder, n *html.Node, depth int) { switch n.Type { case html.TextNode: sb.WriteString(n.Data) return case html.ElementNode: // skip invisible/structural elements switch n.DataAtom { case atom.Script, atom.Style, atom.Nav, atom.Header, atom.Footer, atom.Form, atom.Button, atom.Input, atom.Iframe, atom.Noscript, atom.Aside: return } } if n.Type != html.ElementNode { for c := n.FirstChild; c != nil; c = c.NextSibling { renderMarkdown(sb, c, depth) } return } switch n.DataAtom { case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6: var level int switch n.DataAtom { case atom.H1: level = 1 case atom.H2: level = 2 case atom.H3: level = 3 case atom.H4: level = 4 case atom.H5: level = 5 default: level = 6 } sb.WriteString("\n" + strings.Repeat("#", level) + " ") renderChildren(sb, n, depth) sb.WriteString("\n\n") case atom.P: sb.WriteString("\n") renderChildren(sb, n, depth) sb.WriteString("\n\n") case atom.Br: sb.WriteString("\n") case atom.Hr: sb.WriteString("\n---\n\n") case atom.A: href := attr(n, "href") var inner strings.Builder renderChildren(&inner, n, depth) text := strings.TrimSpace(inner.String()) if text == "" { return // icon-only link — skip (social share buttons, etc.) } if href == "" || href == text { sb.WriteString(text) } else { fmt.Fprintf(sb, "[%s](%s)", text, href) } case atom.Strong, atom.B: sb.WriteString("**") renderChildren(sb, n, depth) sb.WriteString("**") case atom.Em, atom.I: sb.WriteString("*") renderChildren(sb, n, depth) sb.WriteString("*") case atom.Code: // Check if parent is
 — handled there.
		if n.Parent != nil && n.Parent.DataAtom == atom.Pre {
			renderChildren(sb, n, depth)
			return
		}
		sb.WriteString("`")
		renderChildren(sb, n, depth)
		sb.WriteString("`")

	case atom.Pre:
		sb.WriteString("\n```\n")
		renderChildren(sb, n, depth)
		sb.WriteString("\n```\n\n")

	case atom.Blockquote:
		var inner strings.Builder
		renderChildren(&inner, n, depth+1)
		for _, line := range strings.Split(strings.TrimRight(inner.String(), "\n"), "\n") {
			fmt.Fprintf(sb, "> %s\n", line)
		}
		sb.WriteString("\n")

	case atom.Ul:
		sb.WriteString("\n")
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.DataAtom == atom.Li {
				var inner strings.Builder
				renderChildren(&inner, c, depth)
				text := strings.TrimSpace(inner.String())
				fmt.Fprintf(sb, "%s- %s\n", strings.Repeat("  ", depth), text)
			}
		}
		sb.WriteString("\n")

	case atom.Ol:
		sb.WriteString("\n")
		i := 1
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.DataAtom == atom.Li {
				var inner strings.Builder
				renderChildren(&inner, c, depth)
				text := strings.TrimSpace(inner.String())
				fmt.Fprintf(sb, "%s%d. %s\n", strings.Repeat("  ", depth), i, text)
				i++
			}
		}
		sb.WriteString("\n")

	case atom.Img:
		alt := attr(n, "alt")
		src := attr(n, "src")
		if alt != "" || src != "" {
			fmt.Fprintf(sb, "![%s](%s)", alt, src)
		}

	default:
		renderChildren(sb, n, depth)
	}
}

func renderChildren(sb *strings.Builder, n *html.Node, depth int) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		renderMarkdown(sb, c, depth)
	}
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

func findNode(n *html.Node, a atom.Atom) *html.Node {
	if n.Type == html.ElementNode && n.DataAtom == a {
		return n
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if found := findNode(c, a); found != nil {
			return found
		}
	}
	return nil
}

func attr(n *html.Node, key string) string {
	for _, a := range n.Attr {
		if a.Key == key {
			return a.Val
		}
	}
	return ""
}

var multiNewline = regexp.MustCompile(`\n{3,}`)

func cleanupMarkdown(s string) string {
	s = multiNewline.ReplaceAllString(s, "\n\n")
	return strings.TrimSpace(s) + "\n"
}