package main import ( "bytes" "encoding/xml" "fmt" "io" "net/http" "regexp" "strings" "time" ) // nsDecl strips xmlns and xmlns:prefix declarations (both quote styles). // Go's encoding/xml only matches unqualified tags against elements with no // namespace; Atom feeds put every element in the Atom namespace, so without // stripping the declaration none of our struct field tags would match. var nsDecl = regexp.MustCompile(`\s+xmlns(?::\w+)?=(?:"[^"]*"|'[^']*')`) // nsPrefix strips namespace prefixes from element start/end tags // (e.g. → , → ). // After stripping xmlns declarations the prefixes become undeclared; Go's // xml decoder may reject them even in non-strict mode. var nsPrefix = regexp.MustCompile(`( 4096 { summary = truncateUTF8(summary, 4096) } result.Entries = append(result.Entries, &Entry{ GUID: guid, Timestamp: ts, Title: strings.TrimSpace(item.Title), URL: strings.TrimSpace(item.Link), Summary: summary, FetchedAt: fetchedAt, }) } return result, nil } // RSS 1.0 / RDF Site Summary structures. // After namespace stripping, the root is , holds the feed // title, and elements are direct children of (not of ). // dc:date is stripped to ; dc:description may appear too but we use // which is in the core RSS 1.0 namespace. type rdf1Feed struct { XMLName xml.Name `xml:"RDF"` Channel rdf1Channel `xml:"channel"` Items []rdf1Item `xml:"item"` } type rdf1Channel struct { Title string `xml:"title"` } type rdf1Item struct { Title string `xml:"title"` Link string `xml:"link"` Description string `xml:"description"` Date string `xml:"date"` // dc:date → Identifier string `xml:"identifier"` // dc:identifier → } func parseRDF1(data []byte) (*FetchResult, error) { var feed rdf1Feed dec := xml.NewDecoder(bytes.NewReader(data)) dec.Strict = false dec.Entity = xml.HTMLEntity if err := dec.Decode(&feed); err != nil { return nil, fmt.Errorf("RSS 1.0 decode: %w", err) } fetchedAt := time.Now().UTC().Format(time.RFC3339) result := &FetchResult{FeedTitle: strings.TrimSpace(feed.Channel.Title)} for _, item := range feed.Items { ts := parseRSSDate(item.Date) if ts == "" { ts = fetchedAt } guid := strings.TrimSpace(item.Identifier) if guid == "" { guid = synthGUID(item.Link, item.Title) } summary := strings.TrimSpace(item.Description) if len(summary) > 4096 { summary = truncateUTF8(summary, 4096) } result.Entries = append(result.Entries, &Entry{ GUID: guid, Timestamp: ts, Title: strings.TrimSpace(item.Title), URL: strings.TrimSpace(item.Link), Summary: summary, FetchedAt: fetchedAt, }) } return result, nil } func parseAtom(data []byte) (*FetchResult, error) { var feed atomFeed dec := xml.NewDecoder(bytes.NewReader(data)) dec.Strict = false dec.Entity = xml.HTMLEntity if err := dec.Decode(&feed); err != nil { return nil, fmt.Errorf("Atom decode: %w", err) } fetchedAt := time.Now().UTC().Format(time.RFC3339) result := &FetchResult{FeedTitle: strings.TrimSpace(feed.Title.Value)} for _, e := range feed.Entries { ts := strings.TrimSpace(e.Updated) if ts == "" { ts = fetchedAt } else { ts = normalizeTimestamp(ts, fetchedAt) } link := "" for _, l := range e.Links { if l.Rel == "alternate" || l.Rel == "" { link = l.Href break } } guid := strings.TrimSpace(e.ID) if guid == "" { guid = synthGUID(link, e.Title.Value) } summary := strings.TrimSpace(e.Summary.Value) if summary == "" { summary = strings.TrimSpace(e.Content.Value) } if len(summary) > 4096 { summary = truncateUTF8(summary, 4096) } result.Entries = append(result.Entries, &Entry{ GUID: guid, Timestamp: ts, Title: strings.TrimSpace(e.Title.Value), URL: link, Summary: summary, FetchedAt: fetchedAt, }) } return result, nil } // --------------------------------------------------------------------------- // Date helpers // --------------------------------------------------------------------------- var rssDateFormats = []string{ time.RFC1123Z, time.RFC1123, "Mon, 2 Jan 2006 15:04:05 -0700", "Mon, 2 Jan 2006 15:04:05 MST", "2006-01-02T15:04:05Z07:00", "2006-01-02T15:04:05Z", "2006-01-02", } func parseRSSDate(s string) string { s = strings.TrimSpace(s) if s == "" { return "" } now := time.Now().UTC() for _, f := range rssDateFormats { if t, err := time.Parse(f, s); err == nil { if t.After(now) { t = now } return t.UTC().Format(time.RFC3339) } } return "" } func normalizeTimestamp(s, fallback string) string { s = strings.TrimSpace(s) now := time.Now().UTC() for _, f := range rssDateFormats { if t, err := time.Parse(f, s); err == nil { if t.After(now) { t = now } return t.UTC().Format(time.RFC3339) } } return fallback }