summaryrefslogtreecommitdiffci
path: root/internal/blog/s3.go
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat (limited to 'internal/blog/s3.go')
-rw-r--r--internal/blog/s3.go194
1 files changed, 57 insertions, 137 deletions
diff --git a/internal/blog/s3.go b/internal/blog/s3.go
index 3c1a932..86a6463 100644
--- a/internal/blog/s3.go
+++ b/internal/blog/s3.go
@@ -1,16 +1,15 @@
package blog
import (
- "bytes"
"context"
- "encoding/json"
"fmt"
"io"
+ "slices"
"strings"
+ "time"
"github.com/SayaAndy/saya-today-web/config"
"github.com/SayaAndy/saya-today-web/internal/frontmatter"
- "github.com/SayaAndy/saya-today-web/l10n"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
@@ -51,7 +50,6 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) {
}
s3Opts = append(s3Opts, func(o *s3.Options) {
o.UsePathStyle = s3cfg.UsePathStyle
- o.DisableLogOutputChecksumValidationSkipped = true
})
s3cl := s3.NewFromConfig(awsCfg, s3Opts...)
@@ -59,120 +57,76 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) {
return &S3Client{s3cfg.Prefix, s3cfg.BucketName, s3cl}, nil
}
-func (c *S3Client) GetMedleys() ([]MedleyEntry, error) {
- idxRaw, err := c.readAll(MedleysIndexFileName)
- if err != nil {
- return nil, fmt.Errorf("read %s: %w", MedleysIndexFileName, err)
- }
-
- var idx []MedleyEntry
- if err := json.Unmarshal(idxRaw, &idx); err != nil {
- return nil, fmt.Errorf("unmarshal %s: %w", MedleysIndexFileName, err)
- }
-
- return idx, nil
-}
-
func (c *S3Client) Scan(prefix string) ([]*Page, error) {
- out, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(IndexFileName),
- })
- if err != nil {
- return nil, fmt.Errorf("get %s: %w", IndexFileName, err)
- }
- defer out.Body.Close()
+ pages := []*Page{}
- raw, err := io.ReadAll(out.Body)
- if err != nil {
- return nil, fmt.Errorf("read %s: %w", IndexFileName, err)
+ fullPrefix := c.prefix + prefix
+ input := &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucketName),
+ Prefix: aws.String(fullPrefix),
}
- var idx Index
- if err := json.Unmarshal(raw, &idx); err != nil {
- return nil, fmt.Errorf("unmarshal %s: %w", IndexFileName, err)
- }
+ paginator := s3.NewListObjectsV2Paginator(c.s3cl, input)
- wantLang := ""
- if i := strings.Index(prefix, "/"); i > 0 {
- wantLang = prefix[:i]
- }
+ for paginator.HasMorePages() {
+ output, err := paginator.NextPage(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("list S3 objects: %w", err)
+ }
- fullPrefix := c.prefix + prefix
- pages := make([]*Page, 0)
+ for _, obj := range output.Contents {
+ key := aws.ToString(obj.Key)
- switch idx.SchemaVersion {
- case 1:
- for catKey, cat := range *idx.Categories.(*map[string]*IndexV1Category) {
- lang, ok := strings.CutPrefix(catKey, c.prefix)
- if !ok {
- continue
- }
- if wantLang != "" && wantLang != lang {
+ if !strings.HasSuffix(key, ".md") {
continue
}
- for _, e := range cat.Pages {
- if !strings.HasPrefix(e.Link, fullPrefix) {
- continue
- }
- fileName := e.Link[strings.LastIndex(e.Link, "/")+1 : strings.LastIndex(e.Link, ".")]
- pages = append(pages, &Page{
- Link: e.Link,
- FileName: fileName,
- Lang: lang,
- ModifiedTime: e.ModifiedTime,
- Metadata: &frontmatter.Metadata{
- Title: e.Title,
- ShortDescription: e.ShortDescription,
- ActionDate: e.ActionDate,
- PublishedTime: e.PublishedTime,
- Thumbnail: e.Thumbnail,
- Tags: e.Tags,
- Geolocation: e.Geolocation,
- Medley: e.Medley,
- MedleyPart: e.MedleyPart,
- },
- })
+
+ head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("head S3 object %s: %w", key, err)
}
- }
- case 2:
- for catKey, cat := range *idx.Categories.(*map[string]*IndexV2Category) {
- lang, ok := strings.CutPrefix(catKey, c.prefix)
- if !ok {
+
+ if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") {
continue
}
- if wantLang != "" && wantLang != lang {
+
+ meta := head.Metadata
+ if meta["title"] == "" {
continue
}
- for codename, e := range cat.Pages {
- if !strings.HasPrefix(e.Link, fullPrefix) {
- continue
- }
- pages = append(pages, &Page{
- Link: e.Link,
- FileName: codename,
- Lang: lang,
- ModifiedTime: e.ModifiedTime,
- Metadata: &frontmatter.Metadata{
- Title: e.Title,
- ShortDescription: e.ShortDescription,
- ActionDate: e.ActionDate,
- PublishedTime: e.PublishedTime,
- Thumbnail: e.Thumbnail,
- Tags: e.Tags,
- Geolocation: e.Geolocation,
- Medley: e.Medley,
- MedleyPart: e.MedleyPart,
- },
- })
+
+ publishedTime, err := time.Parse(time.RFC3339, meta["published-time"])
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse published time metadata field: %w", err)
}
- }
- }
- medleys, _ := c.GetMedleys()
- for _, medley := range medleys {
- for locale, localname := range medley.Localnames {
- l10n.T.SetPath(localname, true, locale, "Medleys", medley.Codename)
+ linkParts := strings.Split(key, "/")
+ nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
+ fileName := strings.Join(nameParts[:len(nameParts)-1], ".")
+
+ tags := strings.Split(meta["tags"], ",")
+ slices.Sort(tags)
+
+ lang, _ := strings.CutPrefix(linkParts[0], c.prefix)
+
+ pages = append(pages, &Page{
+ Link: key,
+ FileName: fileName,
+ Lang: lang,
+ ModifiedTime: aws.ToTime(obj.LastModified),
+ Metadata: &frontmatter.Metadata{
+ Title: meta["title"],
+ ShortDescription: meta["short-description"],
+ ActionDate: meta["action-date"],
+ PublishedTime: publishedTime,
+ Thumbnail: meta["thumbnail"],
+ Tags: tags,
+ Geolocation: meta["geolocation"],
+ },
+ })
}
}
@@ -180,13 +134,9 @@ func (c *S3Client) Scan(prefix string) ([]*Page, error) {
}
func (c *S3Client) ReadAll(path string) ([]byte, error) {
- return c.readAll(c.prefix + path)
-}
-
-func (c *S3Client) readAll(path string) ([]byte, error) {
output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
Bucket: aws.String(c.bucketName),
- Key: aws.String(path),
+ Key: aws.String(c.prefix + path),
})
if err != nil {
return nil, fmt.Errorf("get S3 object: %w", err)
@@ -202,40 +152,10 @@ func (c *S3Client) readAll(path string) ([]byte, error) {
}
func (c *S3Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) {
- idxRaw, err := c.readAll(IndexFileName)
- if err != nil {
- return nil, nil, fmt.Errorf("read %s: %w", IndexFileName, err)
- }
-
- var idx Index
- if err := json.Unmarshal(idxRaw, &idx); err != nil {
- return nil, nil, fmt.Errorf("unmarshal %s: %w", IndexFileName, err)
- }
-
contentBytes, err := c.ReadAll(path)
if err != nil {
return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err)
}
- switch idx.SchemaVersion {
- case 1:
- return frontmatter.ParseFrontmatter(contentBytes)
- case 2:
- fullPath := c.prefix + path
- page := (*idx.Categories.(*map[string]*IndexV2Category))[fullPath[:strings.LastIndex(fullPath, "/")]].Pages[fullPath[strings.LastIndex(fullPath, "/")+1:strings.LastIndex(fullPath, ".")]]
- metadata = page.Metadata()
-
- if !bytes.HasPrefix(contentBytes, []byte("---\n")) {
- return metadata, contentBytes, nil
- }
-
- end := bytes.Index(contentBytes[4:], []byte("\n---\n"))
- if end == -1 {
- return metadata, contentBytes, nil
- }
-
- return metadata, contentBytes[end+9:], nil
- }
-
return frontmatter.ParseFrontmatter(contentBytes)
}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-20 17:14:58 +0000