feat: move from b2 to s3 for blog pages & facts

feat: use centralized index.json for blog pages, keep paginator as fallback
feat: multithreaded paginator

1 files changed, 148 insertions, 24 deletions

diff --git a/internal/blog/s3.go b/internal/blog/s3.go
index 86a6463..06630ef 100644
--- a/internal/blog/s3.go
+++ b/internal/blog/s3.go
@@ -2,10 +2,15 @@ package blog
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
+	"log/slog"
+	"net/url"
 	"slices"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/SayaAndy/saya-today-web/config"
@@ -14,8 +19,11 @@ import (
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
+const s3ScanConcurrency = 32
+
 type S3Client struct {
 	prefix     string
 	bucketName string
@@ -58,79 +66,195 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) {
 }
 
 func (c *S3Client) Scan(prefix string) ([]*Page, error) {
-	pages := []*Page{}
+	pages, err := c.scanFromIndex(prefix)
+	if err == nil {
+		return pages, nil
+	}
 
-	fullPrefix := c.prefix + prefix
-	input := &s3.ListObjectsV2Input{
+	var nsk *s3types.NoSuchKey
+	if !errors.As(err, &nsk) {
+		return nil, err
+	}
+
+	slog.Warn("index.json missing, falling back to listing", slog.String("prefix", c.prefix))
+	return c.scanByListing(prefix)
+}
+
+func (c *S3Client) scanFromIndex(prefix string) ([]*Page, error) {
+	out, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
 		Bucket: aws.String(c.bucketName),
-		Prefix: aws.String(fullPrefix),
+		Key:    aws.String(IndexFileName),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("get index.json: %w", err)
+	}
+	defer out.Body.Close()
+
+	raw, err := io.ReadAll(out.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read index.json: %w", err)
+	}
+
+	var idx Index
+	if err := json.Unmarshal(raw, &idx); err != nil {
+		return nil, fmt.Errorf("unmarshal index.json: %w", err)
+	}
+
+	wantLang := ""
+	if i := strings.Index(prefix, "/"); i > 0 {
+		wantLang = prefix[:i]
+	}
+
+	fullPrefix := c.prefix + prefix
+	pages := make([]*Page, 0)
+	for catKey, cat := range idx.Categories {
+		lang, ok := strings.CutPrefix(catKey, c.prefix)
+		if !ok {
+			continue
+		}
+		if wantLang != "" && wantLang != lang {
+			continue
+		}
+		for _, e := range cat.Pages {
+			if !strings.HasPrefix(e.Link, fullPrefix) {
+				continue
+			}
+			linkParts := strings.Split(e.Link, "/")
+			nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
+			fileName := strings.Join(nameParts[:len(nameParts)-1], ".")
+			pages = append(pages, &Page{
+				Link:         e.Link,
+				FileName:     fileName,
+				Lang:         lang,
+				ModifiedTime: e.ModifiedTime,
+				Metadata: &frontmatter.Metadata{
+					Title:            e.Title,
+					ShortDescription: e.ShortDescription,
+					ActionDate:       e.ActionDate,
+					PublishedTime:    e.PublishedTime,
+					Thumbnail:        e.Thumbnail,
+					Tags:             e.Tags,
+					Geolocation:      e.Geolocation,
+					Medley:           e.Medley,
+					MedleyPart:       e.MedleyPart,
+				},
+			})
+		}
 	}
+	return pages, nil
+}
 
-	paginator := s3.NewListObjectsV2Paginator(c.s3cl, input)
+func (c *S3Client) scanByListing(prefix string) ([]*Page, error) {
+	fullPrefix := c.prefix + prefix
+
+	type candidate struct {
+		key          string
+		lastModified time.Time
+	}
+	var candidates []candidate
 
+	paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
+		Bucket: aws.String(c.bucketName),
+		Prefix: aws.String(fullPrefix),
+	})
 	for paginator.HasMorePages() {
 		output, err := paginator.NextPage(context.Background())
 		if err != nil {
 			return nil, fmt.Errorf("list S3 objects: %w", err)
 		}
-
 		for _, obj := range output.Contents {
 			key := aws.ToString(obj.Key)
-
 			if !strings.HasSuffix(key, ".md") {
 				continue
 			}
+			candidates = append(candidates, candidate{key, aws.ToTime(obj.LastModified)})
+		}
+	}
+
+	pages := make([]*Page, len(candidates))
+	sem := make(chan struct{}, s3ScanConcurrency)
+	var wg sync.WaitGroup
+	var firstErr error
+	var errMu sync.Mutex
+
+	for i, cand := range candidates {
+		sem <- struct{}{}
+		wg.Go(func() {
+			defer func() { <-sem }()
 
 			head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
 				Bucket: aws.String(c.bucketName),
-				Key:    aws.String(key),
+				Key:    aws.String(cand.key),
 			})
 			if err != nil {
-				return nil, fmt.Errorf("head S3 object %s: %w", key, err)
+				errMu.Lock()
+				if firstErr == nil {
+					firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err)
+				}
+				errMu.Unlock()
+				return
 			}
 
 			if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") {
-				continue
+				return
 			}
-
 			meta := head.Metadata
 			if meta["title"] == "" {
-				continue
+				return
 			}
 
 			publishedTime, err := time.Parse(time.RFC3339, meta["published-time"])
 			if err != nil {
-				return nil, fmt.Errorf("failed to parse published time metadata field: %w", err)
+				errMu.Lock()
+				if firstErr == nil {
+					firstErr = fmt.Errorf("failed to parse published time metadata field: %w", err)
+				}
+				errMu.Unlock()
+				return
 			}
 
-			linkParts := strings.Split(key, "/")
+			linkParts := strings.Split(cand.key, "/")
 			nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
 			fileName := strings.Join(nameParts[:len(nameParts)-1], ".")
+			lang, _ := strings.CutPrefix(linkParts[0], c.prefix)
 
 			tags := strings.Split(meta["tags"], ",")
 			slices.Sort(tags)
 
-			lang, _ := strings.CutPrefix(linkParts[0], c.prefix)
+			title, _ := url.QueryUnescape(meta["title"])
+			shortDescription, _ := url.QueryUnescape(meta["short-description"])
+			thumbnail, _ := url.QueryUnescape(meta["thumbnail"])
 
-			pages = append(pages, &Page{
-				Link:         key,
+			pages[i] = &Page{
+				Link:         cand.key,
 				FileName:     fileName,
 				Lang:         lang,
-				ModifiedTime: aws.ToTime(obj.LastModified),
+				ModifiedTime: cand.lastModified,
 				Metadata: &frontmatter.Metadata{
-					Title:            meta["title"],
-					ShortDescription: meta["short-description"],
+					Title:            title,
+					ShortDescription: shortDescription,
 					ActionDate:       meta["action-date"],
 					PublishedTime:    publishedTime,
-					Thumbnail:        meta["thumbnail"],
+					Thumbnail:        thumbnail,
 					Tags:             tags,
 					Geolocation:      meta["geolocation"],
 				},
-			})
-		}
+			}
+		})
 	}
+	wg.Wait()
 
-	return pages, nil
+	if firstErr != nil {
+		return nil, firstErr
+	}
+
+	out := pages[:0]
+	for _, p := range pages {
+		if p != nil {
+			out = append(out, p)
+		}
+	}
+	return out, nil
 }
 
 func (c *S3Client) ReadAll(path string) ([]byte, error) {