mirror of
https://github.com/R0m1k3/FlowReader.git
synced 2026-10-19 21:28:10 +02:00
fix(backend): harden security and speed up feeds and article API
Security: - WebSocket events are routed to their owner only (no cross-user leak); hub close is idempotent (fixes double-close panic), adds ping/pong and write deadlines. - Session tokens stored as SHA-256 (migration 008 keeps sessions valid); single-query auth middleware puts the user in the request context. - Client IP only trusts X-Forwarded-For from TRUSTED_PROXIES; rate limiter map is bounded; per-user limit on AI summaries. - Argon2id at OWASP minimum with a concurrency cap; constant-time login for unknown emails; atomic first-admin bootstrap; REGISTRATION_ENABLED. - CSP/HSTS/COOP headers, same-origin guard on mutations, body size limits, wider SSRF denylist, bounded feed/page/AI response reads, generic errors. - Upgrade chi, pgx, x/net, x/text, x/crypto (known CVEs); commit go.sum. Performance: - List endpoints return a plain-text excerpt and reading time instead of full HTML; content is sanitized once at ingest (legacy rows backfilled). - Keyset pagination on (sort_at, id) with matching partial indexes; redundant indexes dropped (migration 007). - Fetcher: bounded worker pool, conditional GET (ETag/Last-Modified), exponential backoff, dedupe before insert, column-safe truncation, retention-aware ingest, per-user refresh coalescing. - Read/favorite/read-all are single ownership-scoped statements. - gzip compression, immutable caching for hashed assets, path-safe SPA handler, server timeouts; expired sessions purged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d037e2be34
commit
03e57e4308
40 files changed
+2231
-1898
No files matched your search
@@ -3,6 +3,7 @@ package utils
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -10,6 +11,9 @@ import (
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
// maxPageBytes bounds how much of a remote page is read.
|
||||
const maxPageBytes = 5 << 20
|
||||
|
||||
// ContentExtractor extracts the main text content from a web page.
|
||||
type ContentExtractor struct {
|
||||
client *http.Client
|
||||
@@ -52,7 +56,10 @@ func (e *ContentExtractor) Extract(ctx context.Context, url string) (string, err
|
||||
return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
||||
if resp.ContentLength > maxPageBytes {
|
||||
return "", fmt.Errorf("page too large")
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(io.LimitReader(resp.Body, maxPageBytes))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parsing HTML: %w", err)
|
||||
}
|
||||
@@ -102,9 +109,7 @@ func (e *ContentExtractor) Extract(ctx context.Context, url string) (string, err
|
||||
}
|
||||
|
||||
// Limit to 10000 characters to avoid huge payloads to AI
|
||||
if len(content) > 10000 {
|
||||
content = content[:10000]
|
||||
}
|
||||
content = TruncateRunes(content, 10000)
|
||||
|
||||
return content, nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -39,9 +40,31 @@ func isDisallowedIP(ip net.IP) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if addr, ok := netip.AddrFromSlice(ip); ok {
|
||||
addr = addr.Unmap()
|
||||
for _, p := range extraBlockedPrefixes {
|
||||
if p.Contains(addr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// extraBlockedPrefixes covers reserved ranges and IPv6 transition prefixes
|
||||
// that can embed an internal IPv4 address (NAT64, 6to4, Teredo, IPv4-compatible).
|
||||
var extraBlockedPrefixes = []netip.Prefix{
|
||||
netip.MustParsePrefix("192.0.0.0/24"), // IETF protocol assignments
|
||||
netip.MustParsePrefix("198.18.0.0/15"), // benchmarking
|
||||
netip.MustParsePrefix("240.0.0.0/4"), // reserved + broadcast
|
||||
netip.MustParsePrefix("::/96"), // IPv4-compatible (deprecated)
|
||||
netip.MustParsePrefix("64:ff9b::/96"), // NAT64
|
||||
netip.MustParsePrefix("64:ff9b:1::/48"), // local-use NAT64
|
||||
netip.MustParsePrefix("2002::/16"), // 6to4
|
||||
netip.MustParsePrefix("2001::/32"), // Teredo
|
||||
netip.MustParsePrefix("100::/64"), // discard-only
|
||||
}
|
||||
|
||||
// ValidateExternalURL parses raw, enforces http(s), and verifies that the host
|
||||
// does not resolve to any disallowed (private/internal) address. It returns the
|
||||
// parsed URL so callers can reuse the normalized form.
|
||||
|
||||
@@ -6,16 +6,21 @@ import (
|
||||
)
|
||||
|
||||
// ContentSanitizer handles HTML sanitization for articles.
|
||||
// A bluemonday policy is safe for concurrent use once built.
|
||||
type ContentSanitizer struct {
|
||||
policy *bluemonday.Policy
|
||||
}
|
||||
|
||||
// NewContentSanitizer creates a new sanitizer with a "UGCPolicy" (safe for user-generated content).
|
||||
func NewContentSanitizer() *ContentSanitizer {
|
||||
// Using UGCPolicy allows common tags (b, i, p, img, etc.) but strips dangerous ones.
|
||||
return &ContentSanitizer{
|
||||
policy: bluemonday.UGCPolicy(),
|
||||
}
|
||||
// UGCPolicy allows common tags (b, i, p, img, figure, table…) but strips
|
||||
// scripts, styles, event handlers and non-http(s)/mailto links.
|
||||
policy := bluemonday.UGCPolicy()
|
||||
// Article links open in a new tab instead of navigating away from the
|
||||
// reader (rel="noopener noreferrer nofollow" is added automatically).
|
||||
policy.AddTargetBlankToFullyQualifiedLinks(true)
|
||||
policy.RequireNoReferrerOnFullyQualifiedLinks(true)
|
||||
return &ContentSanitizer{policy: policy}
|
||||
}
|
||||
|
||||
// Sanitize cleans the HTML content.
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// WordsPerMinute is the average silent reading speed for non-fiction
|
||||
// (Brysbaert, 2019) used to estimate reading time.
|
||||
const WordsPerMinute = 238
|
||||
|
||||
// PlainText converts an HTML fragment to whitespace-collapsed plain text.
|
||||
// Content of script/style/noscript elements is dropped and entities are decoded.
|
||||
func PlainText(fragment string) string {
|
||||
if fragment == "" {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
b.Grow(len(fragment) / 2)
|
||||
z := html.NewTokenizer(strings.NewReader(fragment))
|
||||
skip := 0
|
||||
for {
|
||||
switch z.Next() {
|
||||
case html.ErrorToken:
|
||||
return collapseSpaces(b.String())
|
||||
case html.StartTagToken:
|
||||
name, _ := z.TagName()
|
||||
switch string(name) {
|
||||
case "script", "style", "noscript":
|
||||
skip++
|
||||
case "br", "p", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "tr":
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
case html.EndTagToken:
|
||||
name, _ := z.TagName()
|
||||
switch string(name) {
|
||||
case "script", "style", "noscript":
|
||||
if skip > 0 {
|
||||
skip--
|
||||
}
|
||||
case "p", "div", "li", "h1", "h2", "h3", "h4", "h5", "h6", "blockquote", "tr", "td":
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
case html.TextToken:
|
||||
if skip == 0 {
|
||||
b.Write(z.Text())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func collapseSpaces(s string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(s))
|
||||
space := false
|
||||
for _, r := range s {
|
||||
if unicode.IsSpace(r) {
|
||||
space = true
|
||||
continue
|
||||
}
|
||||
if space && b.Len() > 0 {
|
||||
b.WriteByte(' ')
|
||||
}
|
||||
space = false
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// WordCount counts whitespace-separated words in plain text.
|
||||
func WordCount(plain string) int {
|
||||
return len(strings.Fields(plain))
|
||||
}
|
||||
|
||||
// ReadingMinutes converts a word count into a rounded-up reading time (min 1).
|
||||
func ReadingMinutes(words int) int {
|
||||
if words <= 0 {
|
||||
return 1
|
||||
}
|
||||
return (words + WordsPerMinute - 1) / WordsPerMinute
|
||||
}
|
||||
|
||||
// Excerpt returns at most maxRunes runes of plain text, cut on a word
|
||||
// boundary when possible and suffixed with an ellipsis when truncated.
|
||||
func Excerpt(plain string, maxRunes int) string {
|
||||
if utf8.RuneCountInString(plain) <= maxRunes {
|
||||
return plain
|
||||
}
|
||||
cut := TruncateRunes(plain, maxRunes)
|
||||
if i := strings.LastIndexByte(cut, ' '); i > len(cut)*2/3 {
|
||||
cut = cut[:i]
|
||||
}
|
||||
return strings.TrimRight(cut, " ,;:.-–—") + "…"
|
||||
}
|
||||
|
||||
// TruncateRunes truncates s to at most n runes without splitting a UTF-8
|
||||
// sequence.
|
||||
func TruncateRunes(s string, n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
i := 0
|
||||
for pos := range s {
|
||||
if i == n {
|
||||
return s[:pos]
|
||||
}
|
||||
i++
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func TestPlainText(t *testing.T) {
|
||||
got := PlainText(`<p>Bonjour <b>le</b> monde</p><script>alert(1)</script><p>Fin & suite</p>`)
|
||||
want := "Bonjour le monde Fin & suite"
|
||||
if got != want {
|
||||
t.Fatalf("PlainText = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExcerptCutsOnWordBoundary(t *testing.T) {
|
||||
s := strings.Repeat("mot ", 100)
|
||||
got := Excerpt(s, 50)
|
||||
if !strings.HasSuffix(got, "…") || utf8.RuneCountInString(got) > 51 {
|
||||
t.Fatalf("unexpected excerpt %q", got)
|
||||
}
|
||||
if strings.Contains(got, "mo…") {
|
||||
t.Fatalf("excerpt split a word: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTruncateRunesKeepsUTF8Valid(t *testing.T) {
|
||||
got := TruncateRunes("éééé", 2)
|
||||
if got != "éé" || !utf8.ValidString(got) {
|
||||
t.Fatalf("TruncateRunes = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadingMinutes(t *testing.T) {
|
||||
cases := map[int]int{0: 1, 1: 1, 238: 1, 239: 2, 2380: 10}
|
||||
for words, want := range cases {
|
||||
if got := ReadingMinutes(words); got != want {
|
||||
t.Errorf("ReadingMinutes(%d) = %d, want %d", words, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsDisallowedIP(t *testing.T) {
|
||||
blocked := []string{"127.0.0.1", "10.1.2.3", "192.168.1.1", "169.254.169.254", "::1",
|
||||
"64:ff9b::a00:1", "2002:a00:1::1", "198.18.0.1", "100.64.0.1", "::ffff:127.0.0.1"}
|
||||
for _, s := range blocked {
|
||||
if !isDisallowedIP(net.ParseIP(s)) {
|
||||
t.Errorf("%s should be blocked", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []string{"1.1.1.1", "2606:4700:4700::1111"} {
|
||||
if isDisallowedIP(net.ParseIP(s)) {
|
||||
t.Errorf("%s should be allowed", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user