Diffstat (limited to 'internal/blog/s3.go')
| -rw-r--r-- | internal/blog/s3.go | 149 |
1 files changed, 28 insertions, 121 deletions
diff --git a/internal/blog/s3.go b/internal/blog/s3.go index 3c1a932..5ee896a 100644 --- a/internal/blog/s3.go +++ b/internal/blog/s3.go @@ -1,7 +1,6 @@ package blog import ( - "bytes" "context" "encoding/json" "fmt" @@ -10,13 +9,14 @@ import ( "github.com/SayaAndy/saya-today-web/config" "github.com/SayaAndy/saya-today-web/internal/frontmatter" - "github.com/SayaAndy/saya-today-web/l10n" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" ) +const s3ScanConcurrency = 32 + type S3Client struct { prefix string bucketName string @@ -59,20 +59,6 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) { return &S3Client{s3cfg.Prefix, s3cfg.BucketName, s3cl}, nil } -func (c *S3Client) GetMedleys() ([]MedleyEntry, error) { - idxRaw, err := c.readAll(MedleysIndexFileName) - if err != nil { - return nil, fmt.Errorf("read %s: %w", MedleysIndexFileName, err) - } - - var idx []MedleyEntry - if err := json.Unmarshal(idxRaw, &idx); err != nil { - return nil, fmt.Errorf("unmarshal %s: %w", MedleysIndexFileName, err) - } - - return idx, nil -} - func (c *S3Client) Scan(prefix string) ([]*Page, error) { out, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(c.bucketName), @@ -100,79 +86,34 @@ func (c *S3Client) Scan(prefix string) ([]*Page, error) { fullPrefix := c.prefix + prefix pages := make([]*Page, 0) - - switch idx.SchemaVersion { - case 1: - for catKey, cat := range *idx.Categories.(*map[string]*IndexV1Category) { - lang, ok := strings.CutPrefix(catKey, c.prefix) - if !ok { - continue - } - if wantLang != "" && wantLang != lang { - continue - } - for _, e := range cat.Pages { - if !strings.HasPrefix(e.Link, fullPrefix) { - continue - } - fileName := e.Link[strings.LastIndex(e.Link, "/")+1 : strings.LastIndex(e.Link, ".")] - pages = append(pages, &Page{ - Link: e.Link, - FileName: fileName, - Lang: lang, - ModifiedTime: e.ModifiedTime, - Metadata: &frontmatter.Metadata{ - Title: e.Title, - ShortDescription: e.ShortDescription, - ActionDate: e.ActionDate, - PublishedTime: e.PublishedTime, - Thumbnail: e.Thumbnail, - Tags: e.Tags, - Geolocation: e.Geolocation, - Medley: e.Medley, - MedleyPart: e.MedleyPart, - }, - }) - } + for catKey, cat := range idx.Categories { + lang, ok := strings.CutPrefix(catKey, c.prefix) + if !ok { + continue } - case 2: - for catKey, cat := range *idx.Categories.(*map[string]*IndexV2Category) { - lang, ok := strings.CutPrefix(catKey, c.prefix) - if !ok { - continue - } - if wantLang != "" && wantLang != lang { + if wantLang != "" && wantLang != lang { + continue + } + for _, e := range cat.Pages { + if !strings.HasPrefix(e.Link, fullPrefix) { continue } - for codename, e := range cat.Pages { - if !strings.HasPrefix(e.Link, fullPrefix) { - continue - } - pages = append(pages, &Page{ - Link: e.Link, - FileName: codename, - Lang: lang, - ModifiedTime: e.ModifiedTime, - Metadata: &frontmatter.Metadata{ - Title: e.Title, - ShortDescription: e.ShortDescription, - ActionDate: e.ActionDate, - PublishedTime: e.PublishedTime, - Thumbnail: e.Thumbnail, - Tags: e.Tags, - Geolocation: e.Geolocation, - Medley: e.Medley, - MedleyPart: e.MedleyPart, - }, - }) - } - } - } - - medleys, _ := c.GetMedleys() - for _, medley := range medleys { - for locale, localname := range medley.Localnames { - l10n.T.SetPath(localname, true, locale, "Medleys", medley.Codename) + fileName := e.Link[strings.LastIndex(e.Link, "/")+1 : strings.LastIndex(e.Link, ".")] + pages = append(pages, &Page{ + Link: e.Link, + FileName: fileName, + Lang: lang, + ModifiedTime: e.ModifiedTime, + Metadata: &frontmatter.Metadata{ + Title: e.Title, + ShortDescription: e.ShortDescription, + ActionDate: e.ActionDate, + PublishedTime: e.PublishedTime, + Thumbnail: e.Thumbnail, + Tags: e.Tags, + Geolocation: e.Geolocation, + }, + }) } } @@ -180,13 +121,9 @@ func (c *S3Client) Scan(prefix string) ([]*Page, error) { } func (c *S3Client) ReadAll(path string) ([]byte, error) { - return c.readAll(c.prefix + path) -} - -func (c *S3Client) readAll(path string) ([]byte, error) { output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(c.bucketName), - Key: aws.String(path), + Key: aws.String(c.prefix + path), }) if err != nil { return nil, fmt.Errorf("get S3 object: %w", err) @@ -202,40 +139,10 @@ func (c *S3Client) readAll(path string) ([]byte, error) { } func (c *S3Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) { - idxRaw, err := c.readAll(IndexFileName) - if err != nil { - return nil, nil, fmt.Errorf("read %s: %w", IndexFileName, err) - } - - var idx Index - if err := json.Unmarshal(idxRaw, &idx); err != nil { - return nil, nil, fmt.Errorf("unmarshal %s: %w", IndexFileName, err) - } - contentBytes, err := c.ReadAll(path) if err != nil { return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err) } - switch idx.SchemaVersion { - case 1: - return frontmatter.ParseFrontmatter(contentBytes) - case 2: - fullPath := c.prefix + path - page := (*idx.Categories.(*map[string]*IndexV2Category))[fullPath[:strings.LastIndex(fullPath, "/")]].Pages[fullPath[strings.LastIndex(fullPath, "/")+1:strings.LastIndex(fullPath, ".")]] - metadata = page.Metadata() - - if !bytes.HasPrefix(contentBytes, []byte("---\n")) { - return metadata, contentBytes, nil - } - - end := bytes.Index(contentBytes[4:], []byte("\n---\n")) - if end == -1 { - return metadata, contentBytes, nil - } - - return metadata, contentBytes[end+9:], nil - } - return frontmatter.ParseFrontmatter(contentBytes) } |