81 lines
2.0 KiB
Go
81 lines
2.0 KiB
Go
// web_fetch — Fetch an HTTP(S) page and render its HTML as Markdown.
|
|
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
md "github.com/JohannesKaufmann/html-to-markdown"
|
|
)
|
|
|
|
const maxResponseSize = 10 << 20
|
|
|
|
type arguments struct {
|
|
URL string `json:"url"`
|
|
}
|
|
|
|
func main() {
|
|
var args arguments
|
|
if err := json.NewDecoder(os.Stdin).Decode(&args); err != nil {
|
|
fail(err)
|
|
}
|
|
page, err := fetch(args.URL)
|
|
if err != nil {
|
|
fail(err)
|
|
}
|
|
converter := md.NewConverter(args.URL, true, nil)
|
|
markdown, err := converter.ConvertString(page)
|
|
if err != nil {
|
|
fail(fmt.Errorf("convert HTML: %w", err))
|
|
}
|
|
fmt.Print(markdown)
|
|
}
|
|
|
|
func fetch(rawURL string) (string, error) {
|
|
u, err := url.ParseRequestURI(rawURL)
|
|
if err != nil || u.Scheme == "" || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
|
return "", errors.New("url must be an absolute http or https URL")
|
|
}
|
|
|
|
client := &http.Client{Timeout: 30 * time.Second}
|
|
request, err := http.NewRequest(http.MethodGet, u.String(), nil)
|
|
if err != nil {
|
|
return "", fmt.Errorf("create request: %w", err)
|
|
}
|
|
request.Header.Set("Accept", "text/html,application/xhtml+xml")
|
|
request.Header.Set("User-Agent", "ollie-web-fetch/1.0")
|
|
|
|
response, err := client.Do(request)
|
|
if err != nil {
|
|
return "", fmt.Errorf("fetch URL: %w", err)
|
|
}
|
|
defer response.Body.Close()
|
|
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
|
return "", fmt.Errorf("fetch URL: HTTP %s", response.Status)
|
|
}
|
|
|
|
body, err := io.ReadAll(io.LimitReader(response.Body, maxResponseSize+1))
|
|
if err != nil {
|
|
return "", fmt.Errorf("read response: %w", err)
|
|
}
|
|
if len(body) > maxResponseSize {
|
|
return "", fmt.Errorf("response exceeds %d bytes", maxResponseSize)
|
|
}
|
|
if !strings.Contains(strings.ToLower(response.Header.Get("Content-Type")), "text/html") {
|
|
return "", fmt.Errorf("response is not HTML: %s", response.Header.Get("Content-Type"))
|
|
}
|
|
return string(body), nil
|
|
}
|
|
|
|
func fail(err error) {
|
|
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
|
os.Exit(1)
|
|
}
|