diff options
| author | 2026-05-18 01:33:14 +0700 | |
|---|---|---|
| committer | 2026-05-18 01:33:14 +0700 | |
| commit | 2c9d7e7c3110153c025c5324e0c4cc28d58cd2c1 (patch) | |
| tree | 7d091ef937b8389d72ca3b0d28a6abaefb9c083e /internal/blog | |
| parent | 55f0ca98d42b3d540870dc1c9430b9a26ff19807 (diff) | |
| parent | a43cbea2e501371390709005055c019459554bb2 (diff) | |
| download | web-0.16.0.tar.gz web-0.16.0.zip | |
v0.16.0 (#40)v0.16.0
* [feat: all color themes now have light & dark variations depending on
preference](https://github.com/SayaAndy/saya-today-web/commit/44b72cbc80fb89788470bda75eeb9c0bb3dee2f5)
* [feat: use self-hosted pmtiles for
map](https://github.com/SayaAndy/saya-today-web/commit/fa52ed00e8ef3303f14d7c953f1591eb19042635)
* [feat: subheader for blog pages with medley-related
ones](https://github.com/SayaAndy/saya-today-web/commit/c07da125bc5b0bb65f58c4edce912874e46f6ed8)
* feat: color all leaflet elements depending on color theme & scheme
* [feat: combined map
widget](https://github.com/SayaAndy/saya-today-web/pull/40/commits/6418da32d978a8b70d38f1911a154e4dcc33202e)
* feat: migrate static & photos to gcore cdn
* [refactor: rewrite localization for dynamic set and
get](https://github.com/SayaAndy/saya-today-web/commit/e9353a66bf51486f702eaa0f591dea54c9bd4cd1)
* [feat: highlight the blog page visited earlier in blog
catalogue](https://github.com/SayaAndy/saya-today-web/commit/ed476a1f0d3f970f977626688ac316f8b251d3a0)
* fix: inline shadow now sticks to scrolling
* [feat: take all metadata from centralized
index.json](https://github.com/SayaAndy/saya-today-web/commit/1c441f5eab9a2a24fe5309178ce010f32adac1c7)
* [fix: font in like button on smartphone
screens](https://github.com/SayaAndy/saya-today-web/commit/551295ecacebbb94019def022f70397f74c44775)
* [feat: use "page // medley" format for
title](https://github.com/SayaAndy/saya-today-web/pull/40/commits/0dbb85f25c69b5e7929bb832e7967a709a76ff5c)
* feat: multitone backgrounds implementation
* feat: change font to m-plus for short description
* fix: theme wheel position on different dpi screens in global map
* feat: integrate package minor updates
* refactor: convert color vars into oklch
* [refactor: rm scanning by listing for s3
client](https://github.com/SayaAndy/saya-today-web/commit/1a95e20cc4e050244a76a3ea4cccc5ab30ff5a91)
* fix: on errors with blog client init write correct type
Diffstat (limited to 'internal/blog')
| -rw-r--r-- | internal/blog/b2.go | 31 | ||||
| -rw-r--r-- | internal/blog/client.go | 1 | ||||
| -rw-r--r-- | internal/blog/index.go | 83 | ||||
| -rw-r--r-- | internal/blog/s3.go | 269 |
4 files changed, 214 insertions, 170 deletions
diff --git a/internal/blog/b2.go b/internal/blog/b2.go index 0caa0ba..f2ac890 100644 --- a/internal/blog/b2.go +++ b/internal/blog/b2.go @@ -2,6 +2,7 @@ package blog import ( "context" + "encoding/json" "fmt" "slices" "strings" @@ -37,6 +38,20 @@ func NewB2Client(cfg *config.StorageConfig) (Client, error) { return &B2Client{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil } +func (c *B2Client) GetMedleys() ([]MedleyEntry, error) { + idxRaw, err := c.readAll(MedleysIndexFileName) + if err != nil { + return nil, fmt.Errorf("read %s: %w", MedleysIndexFileName, err) + } + + var idx []MedleyEntry + if err := json.Unmarshal(idxRaw, &idx); err != nil { + return nil, fmt.Errorf("unmarshal %s: %w", MedleysIndexFileName, err) + } + + return idx, nil +} + func (c *B2Client) Scan(prefix string) ([]*Page, error) { filePaths := []*Page{} @@ -70,17 +85,15 @@ func (c *B2Client) Scan(prefix string) ([]*Page, error) { return nil, fmt.Errorf("failed to parse published time metadata field: %w", err) } - linkParts := strings.Split(obj.Name(), "/") - nameParts := strings.Split(linkParts[len(linkParts)-1], ".") - fileName := strings.Join(nameParts[:len(linkParts)-1], ".") - + link := obj.Name() + fileName := link[strings.LastIndex(link, "/")+1 : strings.LastIndex(link, ".")] tags := strings.Split(attrs.Info["tags"], ",") slices.Sort(tags) - lang, _ := strings.CutPrefix(linkParts[0], c.prefix) + lang, _ := strings.CutPrefix(link[0:strings.Index(link, "/")], c.prefix) filePaths = append(filePaths, &Page{ - Link: obj.Name(), + Link: link, FileName: fileName, Lang: lang, ModifiedTime: attrs.LastModified, @@ -104,7 +117,11 @@ func (c *B2Client) Scan(prefix string) ([]*Page, error) { } func (c *B2Client) ReadAll(path string) ([]byte, error) { - obj := c.bucket.Object(c.prefix + path) + return c.readAll(c.prefix + path) +} + +func (c *B2Client) readAll(path string) ([]byte, error) { + obj := c.bucket.Object(path) if obj == nil { return nil, fmt.Errorf("failed to reference object in B2 bucket") } diff --git a/internal/blog/client.go b/internal/blog/client.go index 56bbee4..53eb0fd 100644 --- a/internal/blog/client.go +++ b/internal/blog/client.go @@ -17,6 +17,7 @@ type Page struct { type Client interface { Scan(prefix string) ([]*Page, error) + GetMedleys() ([]MedleyEntry, error) ReadAll(path string) ([]byte, error) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) } diff --git a/internal/blog/index.go b/internal/blog/index.go index 861a90e..a09b59e 100644 --- a/internal/blog/index.go +++ b/internal/blog/index.go @@ -1,10 +1,17 @@ package blog -import "time" +import ( + "encoding/json" + "fmt" + "time" + + "github.com/SayaAndy/saya-today-web/internal/frontmatter" +) const IndexFileName = "index.json" +const MedleysIndexFileName = "medleys.json" -const IndexSchemaVersion = 1 +const IndexSchemaVersion = 2 type IndexEntry struct { Link string `json:"link"` @@ -20,13 +27,77 @@ type IndexEntry struct { MedleyPart int `json:"medleyPart,omitempty"` } -type IndexCategory struct { +func (e IndexEntry) Metadata() *frontmatter.Metadata { + return &frontmatter.Metadata{ + Title: e.Title, + ShortDescription: e.ShortDescription, + ActionDate: e.ActionDate, + PublishedTime: e.PublishedTime, + Thumbnail: e.Thumbnail, + Tags: e.Tags, + Geolocation: e.Geolocation, + Medley: e.Medley, + MedleyPart: e.MedleyPart, + } +} + +type IndexV2Category struct { + GeneratedAt time.Time `json:"generatedAt"` + Pages map[string]IndexEntry `json:"pages"` +} + +type IndexV1Category struct { GeneratedAt time.Time `json:"generatedAt"` Pages []IndexEntry `json:"pages"` } type Index struct { - SchemaVersion int `json:"schemaVersion"` - GeneratedAt time.Time `json:"generatedAt"` - Categories map[string]IndexCategory `json:"categories"` + SchemaVersion int `json:"schemaVersion"` + GeneratedAt time.Time `json:"generatedAt"` + Categories any `json:"categories"` +} + +func (idx *Index) UnmarshalJSON(data []byte) error { + var tmp struct { + SchemaVersion int `json:"schemaVersion"` + GeneratedAt time.Time `json:"generatedAt"` + Categories json.RawMessage `json:"categories"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + idx.SchemaVersion = tmp.SchemaVersion + idx.GeneratedAt = tmp.GeneratedAt + + switch tmp.SchemaVersion { + case 1: + var categories map[string]*IndexV1Category + if err := json.Unmarshal(tmp.Categories, &categories); err != nil { + return fmt.Errorf("unmarshal map[string]*IndexV1Category: %w", err) + } + idx.Categories = &categories + case 2: + var categories map[string]*IndexV2Category + if err := json.Unmarshal(tmp.Categories, &categories); err != nil { + return fmt.Errorf("unmarshal map[string]*IndexV2Category: %w", err) + } + idx.Categories = &categories + default: + return fmt.Errorf("unsupported index version: %d", tmp.SchemaVersion) + } + + return nil +} + +type MedleyEntry struct { + Codename string `json:"codename"` + Localnames map[string]string `json:"localnames"` + Content []string `json:"content"` +} + +type MedleyPageEntry struct { + Codename string `json:"codename"` + Position int `json:"position"` } diff --git a/internal/blog/s3.go b/internal/blog/s3.go index bbd5238..3c1a932 100644 --- a/internal/blog/s3.go +++ b/internal/blog/s3.go @@ -1,29 +1,22 @@ package blog import ( + "bytes" "context" "encoding/json" - "errors" "fmt" "io" - "log/slog" - "net/url" - "slices" "strings" - "sync" - "time" "github.com/SayaAndy/saya-today-web/config" "github.com/SayaAndy/saya-today-web/internal/frontmatter" + "github.com/SayaAndy/saya-today-web/l10n" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" - s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" ) -const s3ScanConcurrency = 32 - type S3Client struct { prefix string bucketName string @@ -66,39 +59,38 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) { return &S3Client{s3cfg.Prefix, s3cfg.BucketName, s3cl}, nil } -func (c *S3Client) Scan(prefix string) ([]*Page, error) { - pages, err := c.scanFromIndex(prefix) - if err == nil { - return pages, nil +func (c *S3Client) GetMedleys() ([]MedleyEntry, error) { + idxRaw, err := c.readAll(MedleysIndexFileName) + if err != nil { + return nil, fmt.Errorf("read %s: %w", MedleysIndexFileName, err) } - var nsk *s3types.NoSuchKey - if !errors.As(err, &nsk) { - return nil, err + var idx []MedleyEntry + if err := json.Unmarshal(idxRaw, &idx); err != nil { + return nil, fmt.Errorf("unmarshal %s: %w", MedleysIndexFileName, err) } - slog.Warn("index.json missing, falling back to listing", slog.String("prefix", c.prefix)) - return c.scanByListing(prefix) + return idx, nil } -func (c *S3Client) scanFromIndex(prefix string) ([]*Page, error) { +func (c *S3Client) Scan(prefix string) ([]*Page, error) { out, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(c.bucketName), Key: aws.String(IndexFileName), }) if err != nil { - return nil, fmt.Errorf("get index.json: %w", err) + return nil, fmt.Errorf("get %s: %w", IndexFileName, err) } defer out.Body.Close() raw, err := io.ReadAll(out.Body) if err != nil { - return nil, fmt.Errorf("read index.json: %w", err) + return nil, fmt.Errorf("read %s: %w", IndexFileName, err) } var idx Index if err := json.Unmarshal(raw, &idx); err != nil { - return nil, fmt.Errorf("unmarshal index.json: %w", err) + return nil, fmt.Errorf("unmarshal %s: %w", IndexFileName, err) } wantLang := "" @@ -108,160 +100,93 @@ func (c *S3Client) scanFromIndex(prefix string) ([]*Page, error) { fullPrefix := c.prefix + prefix pages := make([]*Page, 0) - for catKey, cat := range idx.Categories { - lang, ok := strings.CutPrefix(catKey, c.prefix) - if !ok { - continue - } - if wantLang != "" && wantLang != lang { - continue - } - for _, e := range cat.Pages { - if !strings.HasPrefix(e.Link, fullPrefix) { + + switch idx.SchemaVersion { + case 1: + for catKey, cat := range *idx.Categories.(*map[string]*IndexV1Category) { + lang, ok := strings.CutPrefix(catKey, c.prefix) + if !ok { continue } - linkParts := strings.Split(e.Link, "/") - nameParts := strings.Split(linkParts[len(linkParts)-1], ".") - fileName := strings.Join(nameParts[:len(nameParts)-1], ".") - pages = append(pages, &Page{ - Link: e.Link, - FileName: fileName, - Lang: lang, - ModifiedTime: e.ModifiedTime, - Metadata: &frontmatter.Metadata{ - Title: e.Title, - ShortDescription: e.ShortDescription, - ActionDate: e.ActionDate, - PublishedTime: e.PublishedTime, - Thumbnail: e.Thumbnail, - Tags: e.Tags, - Geolocation: e.Geolocation, - Medley: e.Medley, - MedleyPart: e.MedleyPart, - }, - }) - } - } - return pages, nil -} - -func (c *S3Client) scanByListing(prefix string) ([]*Page, error) { - fullPrefix := c.prefix + prefix - - type candidate struct { - key string - lastModified time.Time - } - var candidates []candidate - - paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{ - Bucket: aws.String(c.bucketName), - Prefix: aws.String(fullPrefix), - }) - for paginator.HasMorePages() { - output, err := paginator.NextPage(context.Background()) - if err != nil { - return nil, fmt.Errorf("list S3 objects: %w", err) - } - for _, obj := range output.Contents { - key := aws.ToString(obj.Key) - if !strings.HasSuffix(key, ".md") { + if wantLang != "" && wantLang != lang { continue } - candidates = append(candidates, candidate{key, aws.ToTime(obj.LastModified)}) - } - } - - pages := make([]*Page, len(candidates)) - sem := make(chan struct{}, s3ScanConcurrency) - var wg sync.WaitGroup - var firstErr error - var errMu sync.Mutex - - for i, cand := range candidates { - sem <- struct{}{} - wg.Go(func() { - defer func() { <-sem }() - - head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ - Bucket: aws.String(c.bucketName), - Key: aws.String(cand.key), - }) - if err != nil { - errMu.Lock() - if firstErr == nil { - firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err) + for _, e := range cat.Pages { + if !strings.HasPrefix(e.Link, fullPrefix) { + continue } - errMu.Unlock() - return + fileName := e.Link[strings.LastIndex(e.Link, "/")+1 : strings.LastIndex(e.Link, ".")] + pages = append(pages, &Page{ + Link: e.Link, + FileName: fileName, + Lang: lang, + ModifiedTime: e.ModifiedTime, + Metadata: &frontmatter.Metadata{ + Title: e.Title, + ShortDescription: e.ShortDescription, + ActionDate: e.ActionDate, + PublishedTime: e.PublishedTime, + Thumbnail: e.Thumbnail, + Tags: e.Tags, + Geolocation: e.Geolocation, + Medley: e.Medley, + MedleyPart: e.MedleyPart, + }, + }) } - - if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") { - return + } + case 2: + for catKey, cat := range *idx.Categories.(*map[string]*IndexV2Category) { + lang, ok := strings.CutPrefix(catKey, c.prefix) + if !ok { + continue } - meta := head.Metadata - if meta["title"] == "" { - return + if wantLang != "" && wantLang != lang { + continue } - - publishedTime, err := time.Parse(time.RFC3339, meta["published-time"]) - if err != nil { - errMu.Lock() - if firstErr == nil { - firstErr = fmt.Errorf("failed to parse published time metadata field: %w", err) + for codename, e := range cat.Pages { + if !strings.HasPrefix(e.Link, fullPrefix) { + continue } - errMu.Unlock() - return + pages = append(pages, &Page{ + Link: e.Link, + FileName: codename, + Lang: lang, + ModifiedTime: e.ModifiedTime, + Metadata: &frontmatter.Metadata{ + Title: e.Title, + ShortDescription: e.ShortDescription, + ActionDate: e.ActionDate, + PublishedTime: e.PublishedTime, + Thumbnail: e.Thumbnail, + Tags: e.Tags, + Geolocation: e.Geolocation, + Medley: e.Medley, + MedleyPart: e.MedleyPart, + }, + }) } - - linkParts := strings.Split(cand.key, "/") - nameParts := strings.Split(linkParts[len(linkParts)-1], ".") - fileName := strings.Join(nameParts[:len(nameParts)-1], ".") - lang, _ := strings.CutPrefix(linkParts[0], c.prefix) - - tags := strings.Split(meta["tags"], ",") - slices.Sort(tags) - - title, _ := url.QueryUnescape(meta["title"]) - shortDescription, _ := url.QueryUnescape(meta["short-description"]) - thumbnail, _ := url.QueryUnescape(meta["thumbnail"]) - - pages[i] = &Page{ - Link: cand.key, - FileName: fileName, - Lang: lang, - ModifiedTime: cand.lastModified, - Metadata: &frontmatter.Metadata{ - Title: title, - ShortDescription: shortDescription, - ActionDate: meta["action-date"], - PublishedTime: publishedTime, - Thumbnail: thumbnail, - Tags: tags, - Geolocation: meta["geolocation"], - }, - } - }) - } - wg.Wait() - - if firstErr != nil { - return nil, firstErr + } } - out := pages[:0] - for _, p := range pages { - if p != nil { - out = append(out, p) + medleys, _ := c.GetMedleys() + for _, medley := range medleys { + for locale, localname := range medley.Localnames { + l10n.T.SetPath(localname, true, locale, "Medleys", medley.Codename) } } - return out, nil + + return pages, nil } func (c *S3Client) ReadAll(path string) ([]byte, error) { + return c.readAll(c.prefix + path) +} + +func (c *S3Client) readAll(path string) ([]byte, error) { output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(c.bucketName), - Key: aws.String(c.prefix + path), + Key: aws.String(path), }) if err != nil { return nil, fmt.Errorf("get S3 object: %w", err) @@ -277,10 +202,40 @@ func (c *S3Client) ReadAll(path string) ([]byte, error) { } func (c *S3Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) { + idxRaw, err := c.readAll(IndexFileName) + if err != nil { + return nil, nil, fmt.Errorf("read %s: %w", IndexFileName, err) + } + + var idx Index + if err := json.Unmarshal(idxRaw, &idx); err != nil { + return nil, nil, fmt.Errorf("unmarshal %s: %w", IndexFileName, err) + } + contentBytes, err := c.ReadAll(path) if err != nil { return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err) } + switch idx.SchemaVersion { + case 1: + return frontmatter.ParseFrontmatter(contentBytes) + case 2: + fullPath := c.prefix + path + page := (*idx.Categories.(*map[string]*IndexV2Category))[fullPath[:strings.LastIndex(fullPath, "/")]].Pages[fullPath[strings.LastIndex(fullPath, "/")+1:strings.LastIndex(fullPath, ".")]] + metadata = page.Metadata() + + if !bytes.HasPrefix(contentBytes, []byte("---\n")) { + return metadata, contentBytes, nil + } + + end := bytes.Index(contentBytes[4:], []byte("\n---\n")) + if end == -1 { + return metadata, contentBytes, nil + } + + return metadata, contentBytes[end+9:], nil + } + return frontmatter.ParseFrontmatter(contentBytes) } |