summaryrefslogtreecommitdiff
path: root/internal/blog
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat (limited to 'internal/blog')
-rw-r--r--internal/blog/b2.go133
-rw-r--r--internal/blog/client.go27
-rw-r--r--internal/blog/index.go32
-rw-r--r--internal/blog/s3.go286
4 files changed, 0 insertions, 478 deletions
diff --git a/internal/blog/b2.go b/internal/blog/b2.go
deleted file mode 100644
index 0caa0ba..0000000
--- a/internal/blog/b2.go
+++ /dev/null
@@ -1,133 +0,0 @@
-package blog
-
-import (
- "context"
- "fmt"
- "slices"
- "strings"
- "time"
-
- "github.com/Backblaze/blazer/b2"
- "github.com/SayaAndy/saya-today-web/config"
- "github.com/SayaAndy/saya-today-web/internal/frontmatter"
-)
-
-type B2Client struct {
- prefix string
- bucket *b2.Bucket
- b2cl *b2.Client
-}
-
-func NewB2Client(cfg *config.StorageConfig) (Client, error) {
- if cfg.Type != "b2" {
- return nil, fmt.Errorf("invalid storage type for B2InputClient")
- }
- b2cfg := cfg.Config.(*config.B2Config)
-
- b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey)
- if err != nil {
- return nil, err
- }
-
- bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName)
- if err != nil {
- return nil, err
- }
-
- return &B2Client{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
-}
-
-func (c *B2Client) Scan(prefix string) ([]*Page, error) {
- filePaths := []*Page{}
-
- iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix+prefix))
-
- for iter.Next() {
- obj := iter.Object()
- if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
- }
-
- attrs, err := obj.Attrs(context.Background())
- if err != nil {
- return nil, fmt.Errorf("get attributes for object: %w", err)
- }
-
- if attrs.Status != b2.Uploaded {
- continue
- }
-
- if !strings.Contains(attrs.ContentType, "text/markdown") {
- continue
- }
-
- if _, ok := attrs.Info["title"]; !ok {
- continue
- }
-
- publishedTime, err := time.Parse(time.RFC3339, attrs.Info["published-time"])
- if err != nil {
- return nil, fmt.Errorf("failed to parse published time metadata field: %w", err)
- }
-
- linkParts := strings.Split(obj.Name(), "/")
- nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
- fileName := strings.Join(nameParts[:len(linkParts)-1], ".")
-
- tags := strings.Split(attrs.Info["tags"], ",")
- slices.Sort(tags)
-
- lang, _ := strings.CutPrefix(linkParts[0], c.prefix)
-
- filePaths = append(filePaths, &Page{
- Link: obj.Name(),
- FileName: fileName,
- Lang: lang,
- ModifiedTime: attrs.LastModified,
- Metadata: &frontmatter.Metadata{
- Title: attrs.Info["title"],
- ShortDescription: attrs.Info["short-description"],
- ActionDate: attrs.Info["action-date"],
- PublishedTime: publishedTime,
- Thumbnail: attrs.Info["thumbnail"],
- Tags: tags,
- Geolocation: attrs.Info["geolocation"],
- },
- })
- }
-
- if err := iter.Err(); err != nil {
- return nil, fmt.Errorf("iterate over B2 objects: %w", err)
- }
-
- return filePaths, nil
-}
-
-func (c *B2Client) ReadAll(path string) ([]byte, error) {
- obj := c.bucket.Object(c.prefix + path)
- if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
- }
- attrs, err := obj.Attrs(context.Background())
- if err != nil {
- return nil, fmt.Errorf("error getting attributes of an object: %w", err)
- }
-
- content := make([]byte, attrs.Size)
- reader := obj.NewReader(context.Background())
-
- if _, err = reader.Read(content); err != nil {
- return nil, fmt.Errorf("failed to read file content: %w", err)
- }
-
- return content, nil
-}
-
-func (c *B2Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) {
- contentBytes, err := c.ReadAll(path)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err)
- }
-
- return frontmatter.ParseFrontmatter(contentBytes)
-}
diff --git a/internal/blog/client.go b/internal/blog/client.go
deleted file mode 100644
index 56bbee4..0000000
--- a/internal/blog/client.go
+++ /dev/null
@@ -1,27 +0,0 @@
-package blog
-
-import (
- "time"
-
- "github.com/SayaAndy/saya-today-web/config"
- "github.com/SayaAndy/saya-today-web/internal/frontmatter"
-)
-
-type Page struct {
- Link string
- FileName string
- Lang string
- ModifiedTime time.Time
- Metadata *frontmatter.Metadata
-}
-
-type Client interface {
- Scan(prefix string) ([]*Page, error)
- ReadAll(path string) ([]byte, error)
- ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error)
-}
-
-var NewClientMap = map[string]func(*config.StorageConfig) (Client, error){
- "b2": NewB2Client,
- "s3": NewS3Client,
-}
diff --git a/internal/blog/index.go b/internal/blog/index.go
deleted file mode 100644
index 861a90e..0000000
--- a/internal/blog/index.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package blog
-
-import "time"
-
-const IndexFileName = "index.json"
-
-const IndexSchemaVersion = 1
-
-type IndexEntry struct {
- Link string `json:"link"`
- ModifiedTime time.Time `json:"modifiedTime"`
- Title string `json:"title"`
- ShortDescription string `json:"shortDescription"`
- ActionDate string `json:"actionDate"`
- PublishedTime time.Time `json:"publishedTime"`
- Thumbnail string `json:"thumbnail"`
- Tags []string `json:"tags"`
- Geolocation string `json:"geolocation"`
- Medley string `json:"medley,omitempty"`
- MedleyPart int `json:"medleyPart,omitempty"`
-}
-
-type IndexCategory struct {
- GeneratedAt time.Time `json:"generatedAt"`
- Pages []IndexEntry `json:"pages"`
-}
-
-type Index struct {
- SchemaVersion int `json:"schemaVersion"`
- GeneratedAt time.Time `json:"generatedAt"`
- Categories map[string]IndexCategory `json:"categories"`
-}
diff --git a/internal/blog/s3.go b/internal/blog/s3.go
deleted file mode 100644
index bbd5238..0000000
--- a/internal/blog/s3.go
+++ /dev/null
@@ -1,286 +0,0 @@
-package blog
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "log/slog"
- "net/url"
- "slices"
- "strings"
- "sync"
- "time"
-
- "github.com/SayaAndy/saya-today-web/config"
- "github.com/SayaAndy/saya-today-web/internal/frontmatter"
- "github.com/aws/aws-sdk-go-v2/aws"
- awsconfig "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
- s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
-)
-
-const s3ScanConcurrency = 32
-
-type S3Client struct {
- prefix string
- bucketName string
- s3cl *s3.Client
-}
-
-func NewS3Client(cfg *config.StorageConfig) (Client, error) {
- if cfg.Type != "s3" {
- return nil, fmt.Errorf("invalid storage type for S3Client")
- }
- s3cfg := cfg.Config.(*config.S3Config)
-
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(s3cfg.Region),
- }
- if s3cfg.AccessKeyID != "" && s3cfg.SecretAccessKey != "" {
- opts = append(opts, awsconfig.WithCredentialsProvider(
- credentials.NewStaticCredentialsProvider(s3cfg.AccessKeyID, s3cfg.SecretAccessKey, ""),
- ))
- }
-
- awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
- if err != nil {
- return nil, fmt.Errorf("load AWS config: %w", err)
- }
-
- var s3Opts []func(*s3.Options)
- if s3cfg.Endpoint != "" {
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.BaseEndpoint = aws.String(s3cfg.Endpoint)
- })
- }
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.UsePathStyle = s3cfg.UsePathStyle
- o.DisableLogOutputChecksumValidationSkipped = true
- })
-
- s3cl := s3.NewFromConfig(awsCfg, s3Opts...)
-
- return &S3Client{s3cfg.Prefix, s3cfg.BucketName, s3cl}, nil
-}
-
-func (c *S3Client) Scan(prefix string) ([]*Page, error) {
- pages, err := c.scanFromIndex(prefix)
- if err == nil {
- return pages, nil
- }
-
- var nsk *s3types.NoSuchKey
- if !errors.As(err, &nsk) {
- return nil, err
- }
-
- slog.Warn("index.json missing, falling back to listing", slog.String("prefix", c.prefix))
- return c.scanByListing(prefix)
-}
-
-func (c *S3Client) scanFromIndex(prefix string) ([]*Page, error) {
- out, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(IndexFileName),
- })
- if err != nil {
- return nil, fmt.Errorf("get index.json: %w", err)
- }
- defer out.Body.Close()
-
- raw, err := io.ReadAll(out.Body)
- if err != nil {
- return nil, fmt.Errorf("read index.json: %w", err)
- }
-
- var idx Index
- if err := json.Unmarshal(raw, &idx); err != nil {
- return nil, fmt.Errorf("unmarshal index.json: %w", err)
- }
-
- wantLang := ""
- if i := strings.Index(prefix, "/"); i > 0 {
- wantLang = prefix[:i]
- }
-
- fullPrefix := c.prefix + prefix
- pages := make([]*Page, 0)
- for catKey, cat := range idx.Categories {
- lang, ok := strings.CutPrefix(catKey, c.prefix)
- if !ok {
- continue
- }
- if wantLang != "" && wantLang != lang {
- continue
- }
- for _, e := range cat.Pages {
- if !strings.HasPrefix(e.Link, fullPrefix) {
- continue
- }
- linkParts := strings.Split(e.Link, "/")
- nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
- fileName := strings.Join(nameParts[:len(nameParts)-1], ".")
- pages = append(pages, &Page{
- Link: e.Link,
- FileName: fileName,
- Lang: lang,
- ModifiedTime: e.ModifiedTime,
- Metadata: &frontmatter.Metadata{
- Title: e.Title,
- ShortDescription: e.ShortDescription,
- ActionDate: e.ActionDate,
- PublishedTime: e.PublishedTime,
- Thumbnail: e.Thumbnail,
- Tags: e.Tags,
- Geolocation: e.Geolocation,
- Medley: e.Medley,
- MedleyPart: e.MedleyPart,
- },
- })
- }
- }
- return pages, nil
-}
-
-func (c *S3Client) scanByListing(prefix string) ([]*Page, error) {
- fullPrefix := c.prefix + prefix
-
- type candidate struct {
- key string
- lastModified time.Time
- }
- var candidates []candidate
-
- paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
- Bucket: aws.String(c.bucketName),
- Prefix: aws.String(fullPrefix),
- })
- for paginator.HasMorePages() {
- output, err := paginator.NextPage(context.Background())
- if err != nil {
- return nil, fmt.Errorf("list S3 objects: %w", err)
- }
- for _, obj := range output.Contents {
- key := aws.ToString(obj.Key)
- if !strings.HasSuffix(key, ".md") {
- continue
- }
- candidates = append(candidates, candidate{key, aws.ToTime(obj.LastModified)})
- }
- }
-
- pages := make([]*Page, len(candidates))
- sem := make(chan struct{}, s3ScanConcurrency)
- var wg sync.WaitGroup
- var firstErr error
- var errMu sync.Mutex
-
- for i, cand := range candidates {
- sem <- struct{}{}
- wg.Go(func() {
- defer func() { <-sem }()
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(cand.key),
- })
- if err != nil {
- errMu.Lock()
- if firstErr == nil {
- firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err)
- }
- errMu.Unlock()
- return
- }
-
- if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") {
- return
- }
- meta := head.Metadata
- if meta["title"] == "" {
- return
- }
-
- publishedTime, err := time.Parse(time.RFC3339, meta["published-time"])
- if err != nil {
- errMu.Lock()
- if firstErr == nil {
- firstErr = fmt.Errorf("failed to parse published time metadata field: %w", err)
- }
- errMu.Unlock()
- return
- }
-
- linkParts := strings.Split(cand.key, "/")
- nameParts := strings.Split(linkParts[len(linkParts)-1], ".")
- fileName := strings.Join(nameParts[:len(nameParts)-1], ".")
- lang, _ := strings.CutPrefix(linkParts[0], c.prefix)
-
- tags := strings.Split(meta["tags"], ",")
- slices.Sort(tags)
-
- title, _ := url.QueryUnescape(meta["title"])
- shortDescription, _ := url.QueryUnescape(meta["short-description"])
- thumbnail, _ := url.QueryUnescape(meta["thumbnail"])
-
- pages[i] = &Page{
- Link: cand.key,
- FileName: fileName,
- Lang: lang,
- ModifiedTime: cand.lastModified,
- Metadata: &frontmatter.Metadata{
- Title: title,
- ShortDescription: shortDescription,
- ActionDate: meta["action-date"],
- PublishedTime: publishedTime,
- Thumbnail: thumbnail,
- Tags: tags,
- Geolocation: meta["geolocation"],
- },
- }
- })
- }
- wg.Wait()
-
- if firstErr != nil {
- return nil, firstErr
- }
-
- out := pages[:0]
- for _, p := range pages {
- if p != nil {
- out = append(out, p)
- }
- }
- return out, nil
-}
-
-func (c *S3Client) ReadAll(path string) ([]byte, error) {
- output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(c.prefix + path),
- })
- if err != nil {
- return nil, fmt.Errorf("get S3 object: %w", err)
- }
- defer output.Body.Close()
-
- content, err := io.ReadAll(output.Body)
- if err != nil {
- return nil, fmt.Errorf("read S3 object body: %w", err)
- }
-
- return content, nil
-}
-
-func (c *S3Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) {
- contentBytes, err := c.ReadAll(path)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err)
- }
-
- return frontmatter.ParseFrontmatter(contentBytes)
-}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-17 02:54:31 +0000