From a3aa1dd07abfbbb27a7dfeeb536a6fcc51709256 Mon Sep 17 00:00:00 2001 From: SayaAndy Date: Wed, 8 Apr 2026 08:22:40 +0700 Subject: feat: update all github actions, golang dependencies, golang to 1.26, alpine to 3.23 --- go.mod | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) (limited to 'go.mod') diff --git a/go.mod b/go.mod index 83f1af9..5f95df5 100644 --- a/go.mod +++ b/go.mod @@ -1,43 +1,40 @@ module github.com/SayaAndy/saya-today-web -go 1.25 +go 1.26 require ( github.com/Backblaze/blazer v0.7.2 - github.com/dgraph-io/ristretto/v2 v2.3.0 - github.com/go-co-op/gocron/v2 v2.17.0 - github.com/go-playground/validator/v10 v10.28.0 - github.com/gofiber/fiber/v2 v2.52.9 - github.com/golang-migrate/migrate/v4 v4.19.0 - github.com/mattn/go-sqlite3 v1.14.32 - github.com/valyala/fasthttp v1.68.0 + github.com/dgraph-io/ristretto/v2 v2.4.0 + github.com/go-co-op/gocron/v2 v2.20.0 + github.com/go-playground/validator/v10 v10.30.2 + github.com/gofiber/fiber/v2 v2.52.12 + github.com/golang-migrate/migrate/v4 v4.19.1 + github.com/mattn/go-sqlite3 v1.14.41 + github.com/valyala/fasthttp v1.70.0 github.com/wneessen/go-mail v0.7.2 - github.com/yuin/goldmark v1.7.13 - golang.org/x/crypto v0.43.0 + github.com/yuin/goldmark v1.8.2 + golang.org/x/crypto v0.49.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/andybalholm/brotli v1.2.0 // indirect + github.com/andybalholm/brotli v1.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/gabriel-vasile/mimetype v1.4.10 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/klauspost/compress v1.18.1 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.22 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect ) -- cgit v1.3.1+13 From 66b2a96fcca113105043be3cb9c49f8ee7fe240c Mon Sep 17 00:00:00 2001 From: SayaAndy Date: Wed, 15 Apr 2026 16:11:51 +0700 Subject: feat: support s3 for blog pages refactor: make blog client an interface feat: claude.md --- CLAUDE.md | 62 +++++++++++ config/config.go | 79 +++++++++++++- go.mod | 12 +++ go.sum | 24 +++++ internal/b2/client.go | 136 ------------------------ internal/blog/b2.go | 133 +++++++++++++++++++++++ internal/blog/client.go | 27 +++++ internal/blog/s3.go | 140 +++++++++++++++++++++++++ internal/blogtrigger/blogtrigger.go | 22 ++-- internal/factgiver/factgiver.go | 10 +- internal/mailer/mailer.go | 4 +- internal/router/handlers/api-v1-blog-search.go | 6 +- internal/router/handlers/api-v1-like-get.go | 2 +- internal/router/handlers/api-v1-like-put.go | 2 +- internal/router/handlers/lang-blog-title.go | 16 +-- internal/router/handlers/lang-blog.go | 8 +- internal/router/handlers/lang-map.go | 8 +- internal/router/handlers/lang-user.go | 2 +- internal/router/router.go | 10 +- 19 files changed, 520 insertions(+), 183 deletions(-) create mode 100644 CLAUDE.md delete mode 100644 internal/b2/client.go create mode 100644 internal/blog/b2.go create mode 100644 internal/blog/client.go create mode 100644 internal/blog/s3.go (limited to 'go.mod') diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..34a12ce --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,62 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build & Run + +```bash +# Build (CGO required for SQLite) +CGO_ENABLED=1 go build -o sayana-web . + +# Run (select config via ENVIRONMENT) +ENVIRONMENT=local ./sayana-web + +# Generate Tailwind CSS +npx @tailwindcss/cli -i ./static/input.css -o ./static/output.css + +# Docker build +docker build -t sayana-web . +``` + +Server listens on `:3000`. No Makefile. No test suite currently. + +## Environment + +Config selected by `ENVIRONMENT` var (`local`/`stage`/`prod`) → `config/config.{env}.yaml`. + +Key env vars: `B2_KEY_ID`, `B2_APPLICATION_KEY`, `MAIL_HOST`, `MAIL_ADDRESS`, `MAIL_USERNAME`, `MAIL_PASSWORD`, `MAIL_SALT`. See `.envrc` for local setup. + +## Architecture + +**Go + Fiber v2** web app serving a multi-language blog/travel site with SSR. + +### Request Flow +1. Fiber router with middleware (CORS, compression, ETags, trailing slash) +2. Routes auto-register via `init()` in `internal/router/handlers/` — each handler file adds to `router.Routes` global slice +3. Handlers implement `Route` interface: `Filter()`, `IsTemplated()`, `Render()`, `RenderBody()` +4. Templated routes render via `TemplateManager` (Go `html/template`) with layout composition +5. Pages cached in Ristretto (512MB in-memory) with TTL; segments loadable via `/api/v1/general-page/:part` + +### Storage +- **Blog content:** Markdown files with YAML frontmatter on B2 or S3 (configurable via `blog.Client` interface) +- **Database:** SQLite3 via `database/sql` (no ORM). Migrations in `migrations/` via golang-migrate +- **Tables:** `blog_likes`, `blog_views`, `user_email_table`, subscription tables +- **Session cache:** SQLite-backed `ClientCache` for client IDs/preferences + +### Key Subsystems +- **Markdown rendering:** Goldmark with custom extensions — `TailwindExtension` (applies Tailwind classes) and `GLightboxExtension` (image galleries) +- **Mailer:** SMTP email with verification codes, subscription management +- **BlogTrigger:** gocron scheduler that detects new posts and sends notification emails +- **i18n:** Language in URL path (`/en/`, `/ru/`), locale YAML files in `locale/` + +### Handler Pattern +Handlers in `internal/router/handlers/` follow naming convention matching routes: +- `lang-blog.go` → `/:lang/blog/` pages +- `api-v1-like-put.go` → `/api/v1/like` PUT endpoint +- Each handler self-registers in `init()`, no central route table to maintain + +## Deployment + +CI/CD via GitHub Actions → Docker image to ghcr.io → Ansible playbook (`deploy/`). +- `stage` branch → stage environment +- Version tags → prod environment diff --git a/config/config.go b/config/config.go index eb4a91d..e48a28b 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,8 @@ package config import ( + "encoding/json" + "fmt" "log/slog" "os" @@ -25,8 +27,72 @@ type BlogPagesConfig struct { } type StorageConfig struct { - Type string `json:"type" yaml:"type"` - Config B2Config `json:"Config" yaml:"config" validate:"required"` + Type string `json:"type" yaml:"type,oneof=b2 s3"` + Config any `json:"Config" yaml:"config" validate:"required"` +} + +func (sc *StorageConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + sc.Type = tmp.Type + + switch tmp.Type { + case "b2": + var b2Config B2Config + if err := json.Unmarshal(tmp.Config, &b2Config); err != nil { + return fmt.Errorf("unmarshal B2Config: %w", err) + } + sc.Config = &b2Config + case "s3": + var s3Config S3Config + if err := json.Unmarshal(tmp.Config, &s3Config); err != nil { + return fmt.Errorf("unmarshal S3Config: %w", err) + } + sc.Config = &s3Config + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +func (sc *StorageConfig) UnmarshalYAML(value *yaml.Node) error { + var tmp struct { + Type string `yaml:"type"` + Config yaml.Node `yaml:"config"` + } + + if err := value.Decode(&tmp); err != nil { + return err + } + + sc.Type = tmp.Type + + switch tmp.Type { + case "b2": + var b2Config B2Config + if err := tmp.Config.Decode(&b2Config); err != nil { + return fmt.Errorf("unmarshal B2Config: %w", err) + } + sc.Config = &b2Config + case "s3": + var s3Config S3Config + if err := tmp.Config.Decode(&s3Config); err != nil { + return fmt.Errorf("unmarshal S3Config: %w", err) + } + sc.Config = &s3Config + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil } type B2Config struct { @@ -37,6 +103,15 @@ type B2Config struct { ApplicationKey string `json:"ApplicationKey" yaml:"applicationKey"` } +type S3Config struct { + BucketName string `json:"BucketName" yaml:"bucketName" validate:"required,min=1"` + Region string `json:"Region" yaml:"region" validate:"required,min=1"` + Prefix string `json:"Prefix" yaml:"prefix"` + Endpoint string `json:"Endpoint" yaml:"endpoint" validate:"required,url"` + AccessKeyID string `json:"AccessKeyID" yaml:"accessKeyID"` + SecretAccessKey string `json:"SecretAccessKey" yaml:"secretAccessKey"` +} + type FactGiverConfig struct { Storage StorageConfig `json:"Storage" yaml:"storage" validate:"required"` FactsFileName string `json:"FactsFileName" yaml:"factsFileName" validate:"required"` diff --git a/go.mod b/go.mod index 5f95df5..228e579 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,18 @@ require ( require ( github.com/andybalholm/brotli v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 7b25a49..50f54cb 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,30 @@ github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQ github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= diff --git a/internal/b2/client.go b/internal/b2/client.go deleted file mode 100644 index fda79e0..0000000 --- a/internal/b2/client.go +++ /dev/null @@ -1,136 +0,0 @@ -package b2 - -import ( - "context" - "fmt" - "slices" - "strings" - "time" - - "github.com/Backblaze/blazer/b2" - "github.com/SayaAndy/saya-today-web/config" - "github.com/SayaAndy/saya-today-web/internal/frontmatter" -) - -type B2Client struct { - prefix string - bucket *b2.Bucket - b2cl *b2.Client -} - -func NewB2Client(cfg *config.B2Config) (*B2Client, error) { - b2cl, err := b2.NewClient(context.Background(), cfg.KeyID, cfg.ApplicationKey) - if err != nil { - return nil, err - } - - bucket, err := b2cl.Bucket(context.Background(), cfg.BucketName) - if err != nil { - return nil, err - } - - return &B2Client{b2cl: b2cl, bucket: bucket, prefix: cfg.Prefix}, nil -} - -type BlogPage struct { - Link string - FileName string - Lang string - ModifiedTime time.Time - Metadata *frontmatter.Metadata -} - -func (c *B2Client) Scan(prefix string) ([]*BlogPage, error) { - filePaths := []*BlogPage{} - - iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix+prefix)) - - for iter.Next() { - obj := iter.Object() - if obj == nil { - return nil, fmt.Errorf("failed to reference object in B2 bucket") - } - - attrs, err := obj.Attrs(context.Background()) - if err != nil { - return nil, fmt.Errorf("get attributes for object: %w", err) - } - - if attrs.Status != b2.Uploaded { - continue - } - - if !strings.Contains(attrs.ContentType, "text/markdown") { - continue - } - - if _, ok := attrs.Info["title"]; !ok { - continue - } - - publishedTime, err := time.Parse(time.RFC3339, attrs.Info["published-time"]) - if err != nil { - return nil, fmt.Errorf("failed to parse published time metadata field: %w", err) - } - - linkParts := strings.Split(obj.Name(), "/") - nameParts := strings.Split(linkParts[len(linkParts)-1], ".") - fileName := strings.Join(nameParts[:len(linkParts)-1], ".") - - tags := strings.Split(attrs.Info["tags"], ",") - slices.Sort(tags) - - lang, _ := strings.CutPrefix(linkParts[0], c.prefix) - - filePaths = append(filePaths, &BlogPage{ - Link: obj.Name(), - FileName: fileName, - Lang: lang, - ModifiedTime: attrs.LastModified, - Metadata: &frontmatter.Metadata{ - Title: attrs.Info["title"], - ShortDescription: attrs.Info["short-description"], - ActionDate: attrs.Info["action-date"], - PublishedTime: publishedTime, - Thumbnail: attrs.Info["thumbnail"], - Tags: tags, - Geolocation: attrs.Info["geolocation"], - }, - }) - } - - if err := iter.Err(); err != nil { - return nil, fmt.Errorf("iterate over B2 objects: %w", err) - } - - return filePaths, nil -} - -func (c *B2Client) ReadAll(path string) ([]byte, error) { - obj := c.bucket.Object(c.prefix + path) - if obj == nil { - return nil, fmt.Errorf("failed to reference object in B2 bucket") - } - attrs, err := obj.Attrs(context.Background()) - if err != nil { - return nil, fmt.Errorf("error getting attributes of an object: %w", err) - } - - content := make([]byte, attrs.Size) - reader := obj.NewReader(context.Background()) - - if _, err = reader.Read(content); err != nil { - return nil, fmt.Errorf("failed to read file content: %w", err) - } - - return content, nil -} - -func (c *B2Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) { - contentBytes, err := c.ReadAll(path) - if err != nil { - return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err) - } - - return frontmatter.ParseFrontmatter(contentBytes) -} diff --git a/internal/blog/b2.go b/internal/blog/b2.go new file mode 100644 index 0000000..0caa0ba --- /dev/null +++ b/internal/blog/b2.go @@ -0,0 +1,133 @@ +package blog + +import ( + "context" + "fmt" + "slices" + "strings" + "time" + + "github.com/Backblaze/blazer/b2" + "github.com/SayaAndy/saya-today-web/config" + "github.com/SayaAndy/saya-today-web/internal/frontmatter" +) + +type B2Client struct { + prefix string + bucket *b2.Bucket + b2cl *b2.Client +} + +func NewB2Client(cfg *config.StorageConfig) (Client, error) { + if cfg.Type != "b2" { + return nil, fmt.Errorf("invalid storage type for B2InputClient") + } + b2cfg := cfg.Config.(*config.B2Config) + + b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) + if err != nil { + return nil, err + } + + bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) + if err != nil { + return nil, err + } + + return &B2Client{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil +} + +func (c *B2Client) Scan(prefix string) ([]*Page, error) { + filePaths := []*Page{} + + iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix+prefix)) + + for iter.Next() { + obj := iter.Object() + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + if attrs.Status != b2.Uploaded { + continue + } + + if !strings.Contains(attrs.ContentType, "text/markdown") { + continue + } + + if _, ok := attrs.Info["title"]; !ok { + continue + } + + publishedTime, err := time.Parse(time.RFC3339, attrs.Info["published-time"]) + if err != nil { + return nil, fmt.Errorf("failed to parse published time metadata field: %w", err) + } + + linkParts := strings.Split(obj.Name(), "/") + nameParts := strings.Split(linkParts[len(linkParts)-1], ".") + fileName := strings.Join(nameParts[:len(linkParts)-1], ".") + + tags := strings.Split(attrs.Info["tags"], ",") + slices.Sort(tags) + + lang, _ := strings.CutPrefix(linkParts[0], c.prefix) + + filePaths = append(filePaths, &Page{ + Link: obj.Name(), + FileName: fileName, + Lang: lang, + ModifiedTime: attrs.LastModified, + Metadata: &frontmatter.Metadata{ + Title: attrs.Info["title"], + ShortDescription: attrs.Info["short-description"], + ActionDate: attrs.Info["action-date"], + PublishedTime: publishedTime, + Thumbnail: attrs.Info["thumbnail"], + Tags: tags, + Geolocation: attrs.Info["geolocation"], + }, + }) + } + + if err := iter.Err(); err != nil { + return nil, fmt.Errorf("iterate over B2 objects: %w", err) + } + + return filePaths, nil +} + +func (c *B2Client) ReadAll(path string) ([]byte, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("error getting attributes of an object: %w", err) + } + + content := make([]byte, attrs.Size) + reader := obj.NewReader(context.Background()) + + if _, err = reader.Read(content); err != nil { + return nil, fmt.Errorf("failed to read file content: %w", err) + } + + return content, nil +} + +func (c *B2Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) { + contentBytes, err := c.ReadAll(path) + if err != nil { + return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err) + } + + return frontmatter.ParseFrontmatter(contentBytes) +} diff --git a/internal/blog/client.go b/internal/blog/client.go new file mode 100644 index 0000000..56bbee4 --- /dev/null +++ b/internal/blog/client.go @@ -0,0 +1,27 @@ +package blog + +import ( + "time" + + "github.com/SayaAndy/saya-today-web/config" + "github.com/SayaAndy/saya-today-web/internal/frontmatter" +) + +type Page struct { + Link string + FileName string + Lang string + ModifiedTime time.Time + Metadata *frontmatter.Metadata +} + +type Client interface { + Scan(prefix string) ([]*Page, error) + ReadAll(path string) ([]byte, error) + ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) +} + +var NewClientMap = map[string]func(*config.StorageConfig) (Client, error){ + "b2": NewB2Client, + "s3": NewS3Client, +} diff --git a/internal/blog/s3.go b/internal/blog/s3.go new file mode 100644 index 0000000..de4eb3a --- /dev/null +++ b/internal/blog/s3.go @@ -0,0 +1,140 @@ +package blog + +import ( + "context" + "fmt" + "io" + "slices" + "strings" + "time" + + "github.com/SayaAndy/saya-today-web/config" + "github.com/SayaAndy/saya-today-web/internal/frontmatter" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type S3Client struct { + prefix string + bucketName string + s3cl *s3.Client +} + +func NewS3Client(cfg *config.StorageConfig) (Client, error) { + if cfg.Type != "s3" { + return nil, fmt.Errorf("invalid storage type for S3Client") + } + s3cfg := cfg.Config.(*config.S3Config) + + s3cl := s3.New(s3.Options{ + Region: s3cfg.Region, + BaseEndpoint: aws.String(s3cfg.Endpoint), + Credentials: credentials.NewStaticCredentialsProvider(s3cfg.AccessKeyID, s3cfg.SecretAccessKey, ""), + }) + + return &S3Client{s3cfg.Prefix, s3cfg.BucketName, s3cl}, nil +} + +func (c *S3Client) Scan(prefix string) ([]*Page, error) { + pages := []*Page{} + + fullPrefix := c.prefix + prefix + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucketName), + Prefix: aws.String(fullPrefix), + } + + paginator := s3.NewListObjectsV2Paginator(c.s3cl, input) + + for paginator.HasMorePages() { + output, err := paginator.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("list S3 objects: %w", err) + } + + for _, obj := range output.Contents { + key := aws.ToString(obj.Key) + + if !strings.HasSuffix(key, ".md") { + continue + } + + head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("head S3 object %s: %w", key, err) + } + + if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") { + continue + } + + meta := head.Metadata + if meta["title"] == "" { + continue + } + + publishedTime, err := time.Parse(time.RFC3339, meta["published-time"]) + if err != nil { + return nil, fmt.Errorf("failed to parse published time metadata field: %w", err) + } + + linkParts := strings.Split(key, "/") + nameParts := strings.Split(linkParts[len(linkParts)-1], ".") + fileName := strings.Join(nameParts[:len(nameParts)-1], ".") + + tags := strings.Split(meta["tags"], ",") + slices.Sort(tags) + + lang, _ := strings.CutPrefix(linkParts[0], c.prefix) + + pages = append(pages, &Page{ + Link: key, + FileName: fileName, + Lang: lang, + ModifiedTime: aws.ToTime(obj.LastModified), + Metadata: &frontmatter.Metadata{ + Title: meta["title"], + ShortDescription: meta["short-description"], + ActionDate: meta["action-date"], + PublishedTime: publishedTime, + Thumbnail: meta["thumbnail"], + Tags: tags, + Geolocation: meta["geolocation"], + }, + }) + } + } + + return pages, nil +} + +func (c *S3Client) ReadAll(path string) ([]byte, error) { + output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(c.prefix + path), + }) + if err != nil { + return nil, fmt.Errorf("get S3 object: %w", err) + } + defer output.Body.Close() + + content, err := io.ReadAll(output.Body) + if err != nil { + return nil, fmt.Errorf("read S3 object body: %w", err) + } + + return content, nil +} + +func (c *S3Client) ReadFrontmatter(path string) (metadata *frontmatter.Metadata, markdown []byte, err error) { + contentBytes, err := c.ReadAll(path) + if err != nil { + return nil, nil, fmt.Errorf("failed to read file for frontmatter parsing: %w", err) + } + + return frontmatter.ParseFrontmatter(contentBytes) +} diff --git a/internal/blogtrigger/blogtrigger.go b/internal/blogtrigger/blogtrigger.go index c5b3097..81efafe 100644 --- a/internal/blogtrigger/blogtrigger.go +++ b/internal/blogtrigger/blogtrigger.go @@ -5,29 +5,29 @@ import ( "log/slog" "github.com/SayaAndy/saya-today-web/config" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" "github.com/go-co-op/gocron/v2" ) type BlogTriggerScheduler struct { s gocron.Scheduler - knownBlogPages map[string]map[string]*b2.BlogPage - b2Client *b2.B2Client - onTrigger func([]*b2.BlogPage) error + knownBlogPages map[string]map[string]*blog.Page + blogClient blog.Client + onTrigger func([]*blog.Page) error } -func NewBlogTriggerScheduler(b2Client *b2.B2Client, availableLanguages []config.AvailableLanguageConfig, cron string, onTrigger func([]*b2.BlogPage) error) (*BlogTriggerScheduler, error) { +func NewBlogTriggerScheduler(blogClient blog.Client, availableLanguages []config.AvailableLanguageConfig, cron string, onTrigger func([]*blog.Page) error) (*BlogTriggerScheduler, error) { s, err := gocron.NewScheduler() if err != nil { return nil, fmt.Errorf("failed to create new scheduler: %w", err) } - knownBlogPages := make(map[string]map[string]*b2.BlogPage, len(availableLanguages)) + knownBlogPages := make(map[string]map[string]*blog.Page, len(availableLanguages)) for _, lang := range availableLanguages { - knownBlogPages[lang.Name] = make(map[string]*b2.BlogPage) + knownBlogPages[lang.Name] = make(map[string]*blog.Page) } - bts := &BlogTriggerScheduler{s, knownBlogPages, b2Client, onTrigger} + bts := &BlogTriggerScheduler{s, knownBlogPages, blogClient, onTrigger} defer bts.s.Start() bts.s.NewJob(gocron.CronJob(cron, false), gocron.NewTask(func(bts *BlogTriggerScheduler) { @@ -53,10 +53,10 @@ func (bts *BlogTriggerScheduler) Close() error { return bts.s.Shutdown() } -func (bts *BlogTriggerScheduler) scan() (newPages []*b2.BlogPage, err error) { - newPages = make([]*b2.BlogPage, 0) +func (bts *BlogTriggerScheduler) scan() (newPages []*blog.Page, err error) { + newPages = make([]*blog.Page, 0) for lang := range bts.knownBlogPages { - posts, err := bts.b2Client.Scan(lang + "/") + posts, err := bts.blogClient.Scan(lang + "/") if err != nil { return nil, fmt.Errorf("failed to scan blog pages in b2 on '%s': %w", lang, err) } diff --git a/internal/factgiver/factgiver.go b/internal/factgiver/factgiver.go index 5b418ee..c5b8bc4 100644 --- a/internal/factgiver/factgiver.go +++ b/internal/factgiver/factgiver.go @@ -8,11 +8,11 @@ import ( "time" "github.com/SayaAndy/saya-today-web/config" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" ) type FactGiver struct { - b2Client *b2.B2Client + blogClient blog.Client cache map[string][]string langs []config.AvailableLanguageConfig factsFileName string @@ -21,13 +21,13 @@ type FactGiver struct { } func NewFactGiver(cfg *config.FactGiverConfig, langs []config.AvailableLanguageConfig) (*FactGiver, error) { - b2Client, err := b2.NewB2Client(&cfg.Storage.Config) + blogClient, err := blog.NewClientMap[cfg.Storage.Type](&cfg.Storage) if err != nil { return nil, fmt.Errorf("fail to init b2 client for a new fact giver: %s", err.Error()) } factGiver := &FactGiver{ - b2Client: b2Client, + blogClient: blogClient, cache: make(map[string][]string, len(langs)), langs: langs, factsFileName: cfg.FactsFileName, @@ -53,7 +53,7 @@ func (g *FactGiver) Give(lang string) [3]string { func (g *FactGiver) initCache() error { for _, lang := range g.langs { localFacts := strings.Replace(g.factsFileName, "*", lang.Name, 1) - factsContentBytes, err := g.b2Client.ReadAll(localFacts) + factsContentBytes, err := g.blogClient.ReadAll(localFacts) if err != nil { return fmt.Errorf("fail to read '%s' facts file: %s", lang.Name, err.Error()) } diff --git a/internal/mailer/mailer.go b/internal/mailer/mailer.go index d6dbd17..ffa7b30 100644 --- a/internal/mailer/mailer.go +++ b/internal/mailer/mailer.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" "github.com/SayaAndy/saya-today-web/internal/templatemanager" "github.com/SayaAndy/saya-today-web/locale" "github.com/dgraph-io/ristretto/v2" @@ -418,7 +418,7 @@ func (m *Mailer) Subscribe(userIdHash []byte, subscriptionType SubscriptionType, return nil } -func (m *Mailer) NewPost(post *b2.BlogPage) error { +func (m *Mailer) NewPost(post *blog.Page) error { tx, err := m.db.Begin() if err != nil { return fmt.Errorf("failed to initialize transaction with db: %s", err) diff --git a/internal/router/handlers/api-v1-blog-search.go b/internal/router/handlers/api-v1-blog-search.go index 1e8a480..c9a91ad 100644 --- a/internal/router/handlers/api-v1-blog-search.go +++ b/internal/router/handlers/api-v1-blog-search.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" "github.com/SayaAndy/saya-today-web/internal/router" "github.com/gofiber/fiber/v2" ) @@ -56,11 +56,11 @@ func (r *BlogSearchHandler) Render(c *fiber.Ctx, supplements *router.Supplements cacheKey := "blog-search." + lang + ".pages-list" - var pages []*b2.BlogPage + var pages []*blog.Page if pagesBytes, ok := supplements.PageCache.Get(cacheKey); pagesBytes != nil || ok { json.Unmarshal(pagesBytes, &pages) } else { - pages, err = supplements.B2Client.Scan(lang + "/") + pages, err = supplements.BlogClient.Scan(lang + "/") if err != nil { return fiber.StatusInternalServerError, fmt.Errorf("failed to scan pages for '%s' lang: %w", lang, err) } diff --git a/internal/router/handlers/api-v1-like-get.go b/internal/router/handlers/api-v1-like-get.go index e9c176e..0eaf5f6 100644 --- a/internal/router/handlers/api-v1-like-get.go +++ b/internal/router/handlers/api-v1-like-get.go @@ -48,7 +48,7 @@ func (r *GetLikeHandler) Render(c *fiber.Ctx, supplements *router.Supplements, l page := pathParts[2] pageLink := lang + "/" + page + ".md" - if pages, _ := supplements.B2Client.Scan(pageLink); len(pages) == 0 { + if pages, _ := supplements.BlogClient.Scan(pageLink); len(pages) == 0 { return fiber.StatusNotFound, fmt.Errorf("server did not find '%s' article", pageLink) } diff --git a/internal/router/handlers/api-v1-like-put.go b/internal/router/handlers/api-v1-like-put.go index ea4ef4b..434f15a 100644 --- a/internal/router/handlers/api-v1-like-put.go +++ b/internal/router/handlers/api-v1-like-put.go @@ -49,7 +49,7 @@ func (r *PutLikeHandler) Render(c *fiber.Ctx, supplements *router.Supplements, l page := pathParts[2] pageLink := lang + "/" + page + ".md" - if pages, _ := supplements.B2Client.Scan(pageLink); len(pages) == 0 { + if pages, _ := supplements.BlogClient.Scan(pageLink); len(pages) == 0 { return fiber.StatusNotFound, fmt.Errorf("server did not find '%s' article", pageLink) } diff --git a/internal/router/handlers/lang-blog-title.go b/internal/router/handlers/lang-blog-title.go index 5bfc6e7..f1be0a5 100644 --- a/internal/router/handlers/lang-blog-title.go +++ b/internal/router/handlers/lang-blog-title.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" "github.com/SayaAndy/saya-today-web/internal/frontmatter" "github.com/SayaAndy/saya-today-web/internal/router" "github.com/gofiber/fiber/v2" @@ -49,7 +49,7 @@ func (r *BlogPageHandler) ToValidateLang() router.LangSetting { func (r *BlogPageHandler) SitemapInfo(supplements *router.Supplements) []router.SitemapInfo { sitemapInfo := []router.SitemapInfo{} - pages, err := supplements.B2Client.Scan("") + pages, err := supplements.BlogClient.Scan("") if err != nil { return sitemapInfo } @@ -66,7 +66,7 @@ func (r *BlogPageHandler) SitemapInfo(supplements *router.Supplements) []router. } func (r *BlogPageHandler) AddMeta(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (meta []router.MetaField, err error) { - metadata, _, err := supplements.B2Client.ReadFrontmatter(lang + "/" + c.Params("title") + ".md") + metadata, _, err := supplements.BlogClient.ReadFrontmatter(lang + "/" + c.Params("title") + ".md") if err != nil { return nil, fmt.Errorf("failed to read frontmatter of the desired blog post: %w", err) } @@ -82,7 +82,7 @@ func (r *BlogPageHandler) AddMeta(c *fiber.Ctx, supplements *router.Supplements, } func (r *BlogPageHandler) AddLinkedData(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (ld map[string]any, err error) { - metadata, _, err := supplements.B2Client.ReadFrontmatter(lang + "/" + c.Params("title") + ".md") + metadata, _, err := supplements.BlogClient.ReadFrontmatter(lang + "/" + c.Params("title") + ".md") if err != nil { return nil, fmt.Errorf("failed to read frontmatter of the desired blog post: %w", err) } @@ -103,7 +103,7 @@ func (r *BlogPageHandler) RenderBody(c *fiber.Ctx, supplements *router.Supplemen return fiber.StatusBadRequest, fmt.Errorf("failed to get path from referer: %w", err) } - metadata, parsedMarkdown, err := readBlogPost(supplements.MarkdownRenderer, supplements.B2Client, lang+"/"+pathParts[2]) + metadata, parsedMarkdown, err := readBlogPost(supplements.MarkdownRenderer, supplements.BlogClient, lang+"/"+pathParts[2]) if err != nil { return fiber.StatusNotFound, fmt.Errorf("failed to find '%s' post: %w", pathParts[2], err) } @@ -138,7 +138,7 @@ func (r *BlogPageHandler) RenderHeader(c *fiber.Ctx, supplements *router.Supplem return fiber.StatusBadRequest, fmt.Errorf("failed to get path from referer: %w", err) } - metadata, _, err := supplements.B2Client.ReadFrontmatter(lang + "/" + pathParts[2] + ".md") + metadata, _, err := supplements.BlogClient.ReadFrontmatter(lang + "/" + pathParts[2] + ".md") if err != nil { return fiber.StatusNotFound, fmt.Errorf("could not read '%s' for metadata: %w", path, err) } @@ -148,8 +148,8 @@ func (r *BlogPageHandler) RenderHeader(c *fiber.Ctx, supplements *router.Supplem return fiber.StatusOK, nil } -func readBlogPost(md goldmark.Markdown, b2Client *b2.B2Client, sourceName string) (metadata *frontmatter.Metadata, html string, err error) { - metadata, markdown, err := b2Client.ReadFrontmatter(sourceName + ".md") +func readBlogPost(md goldmark.Markdown, blogClient blog.Client, sourceName string) (metadata *frontmatter.Metadata, html string, err error) { + metadata, markdown, err := blogClient.ReadFrontmatter(sourceName + ".md") if err != nil { return nil, "", fmt.Errorf("failed to read a frontmatter file: %w", err) } diff --git a/internal/router/handlers/lang-blog.go b/internal/router/handlers/lang-blog.go index 709d3dc..79ec9f3 100644 --- a/internal/router/handlers/lang-blog.go +++ b/internal/router/handlers/lang-blog.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" "github.com/SayaAndy/saya-today-web/internal/router" "github.com/gofiber/fiber/v2" ) @@ -79,7 +79,7 @@ func (r *CatalogueHandler) RenderBody(c *fiber.Ctx, supplements *router.Suppleme queryTags = append(queryTags, string(match[1])) } - tagsArray, err := getTags(supplements.B2Client, lang) + tagsArray, err := getTags(supplements.BlogClient, lang) if err != nil { return fiber.StatusInternalServerError, fmt.Errorf("failed to get the available tags") } @@ -102,8 +102,8 @@ type Tag struct { Count int `json:"Count" yaml:"count"` } -func getTags(b2Client *b2.B2Client, lang string) (tags []Tag, err error) { - pages, err := b2Client.Scan(lang + "/") +func getTags(blogClient blog.Client, lang string) (tags []Tag, err error) { + pages, err := blogClient.Scan(lang + "/") if err != nil { slog.Warn("failed to scan pages via b2", slog.String("error", err.Error())) return nil, fmt.Errorf("failed to scan pages via b2: %w", err) diff --git a/internal/router/handlers/lang-map.go b/internal/router/handlers/lang-map.go index 551b485..d452bc7 100644 --- a/internal/router/handlers/lang-map.go +++ b/internal/router/handlers/lang-map.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" "github.com/SayaAndy/saya-today-web/internal/router" "github.com/gofiber/fiber/v2" ) @@ -49,8 +49,8 @@ func (r *MapHandler) SitemapInfo(supplements *router.Supplements) []router.Sitem } func (r *MapHandler) Render(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) { - pages, err := supplements.B2Client.Scan(lang + "/") - slices.SortFunc(pages, func(a *b2.BlogPage, b *b2.BlogPage) int { + pages, err := supplements.BlogClient.Scan(lang + "/") + slices.SortFunc(pages, func(a *blog.Page, b *blog.Page) int { return a.Metadata.PublishedTime.Compare(b.Metadata.PublishedTime) }) status := fiber.StatusOK @@ -60,7 +60,7 @@ func (r *MapHandler) Render(c *fiber.Ctx, supplements *router.Supplements, lang slog.String("lang", lang), ) status = fiber.StatusPartialContent - pages = []*b2.BlogPage{} + pages = []*blog.Page{} } type MapMarker struct { diff --git a/internal/router/handlers/lang-user.go b/internal/router/handlers/lang-user.go index eae1685..0dda529 100644 --- a/internal/router/handlers/lang-user.go +++ b/internal/router/handlers/lang-user.go @@ -55,7 +55,7 @@ func (r *UserHandler) RenderBody(c *fiber.Ctx, supplements *router.Supplements, slog.Error("get info from mailer about a client", slog.String("error", err.Error())) } - tagsArray, err := getTags(supplements.B2Client, lang) + tagsArray, err := getTags(supplements.BlogClient, lang) if err != nil { return fiber.ErrInternalServerError.Code, fmt.Errorf("failed to get the available tags") } diff --git a/internal/router/router.go b/internal/router/router.go index 3242759..e8fb47a 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -13,7 +13,7 @@ import ( "time" "github.com/SayaAndy/saya-today-web/config" - "github.com/SayaAndy/saya-today-web/internal/b2" + "github.com/SayaAndy/saya-today-web/internal/blog" "github.com/SayaAndy/saya-today-web/internal/blogtrigger" "github.com/SayaAndy/saya-today-web/internal/factgiver" "github.com/SayaAndy/saya-today-web/internal/glightbox" @@ -90,7 +90,7 @@ type Route interface { type Supplements struct { DB *sql.DB - B2Client *b2.B2Client + BlogClient blog.Client Localization map[string]*locale.LocaleConfig AvailableLanguages []config.AvailableLanguageConfig ClientCache *ClientCache @@ -140,7 +140,7 @@ func NewRouter(cfg *config.Config) (*Router, error) { } slog.Debug("successfully applied migrations") - supplements.B2Client, err = b2.NewB2Client(&cfg.BlogPages.Storage.Config) + supplements.BlogClient, err = blog.NewClientMap[cfg.BlogPages.Storage.Type](&cfg.BlogPages.Storage) if err != nil { return nil, fmt.Errorf("fail to initialize b2 client: %w", err) } @@ -198,8 +198,8 @@ func NewRouter(cfg *config.Config) (*Router, error) { return nil, fmt.Errorf("fail to initialize mailer: %w", err) } - supplements.BlogTrigger, err = blogtrigger.NewBlogTriggerScheduler(supplements.B2Client, cfg.AvailableLanguages, cfg.Mail.Trigger.OnNewPost, - func(bp []*b2.BlogPage) error { + supplements.BlogTrigger, err = blogtrigger.NewBlogTriggerScheduler(supplements.BlogClient, cfg.AvailableLanguages, cfg.Mail.Trigger.OnNewPost, + func(bp []*blog.Page) error { for _, post := range bp { if err := supplements.Mailer.NewPost(post); err != nil { return err -- cgit v1.3.1+13 From ab557dd4df732c5d461bf7015f9eb0e761c982b7 Mon Sep 17 00:00:00 2001 From: SayaAndy Date: Wed, 15 Apr 2026 16:31:54 +0700 Subject: feat: add usepathstyle flag for s3 config refactor: use aws-sdk-go-v2/config package for configuring s3 client --- config/config.go | 3 ++- go.mod | 10 ++++++++++ go.sum | 14 ++++++++++++++ internal/blog/s3.go | 29 +++++++++++++++++++++++++---- 4 files changed, 51 insertions(+), 5 deletions(-) (limited to 'go.mod') diff --git a/config/config.go b/config/config.go index e48a28b..ebe0367 100644 --- a/config/config.go +++ b/config/config.go @@ -107,7 +107,8 @@ type S3Config struct { BucketName string `json:"BucketName" yaml:"bucketName" validate:"required,min=1"` Region string `json:"Region" yaml:"region" validate:"required,min=1"` Prefix string `json:"Prefix" yaml:"prefix"` - Endpoint string `json:"Endpoint" yaml:"endpoint" validate:"required,url"` + Endpoint string `json:"Endpoint" yaml:"endpoint" validate:"url"` + UsePathStyle bool `json:"UsePathStyle"` AccessKeyID string `json:"AccessKeyID" yaml:"accessKeyID"` SecretAccessKey string `json:"SecretAccessKey" yaml:"secretAccessKey"` } diff --git a/go.mod b/go.mod index 228e579..373095d 100644 --- a/go.mod +++ b/go.mod @@ -17,10 +17,20 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +require ( + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect +) + require ( github.com/andybalholm/brotli v1.2.1 // indirect github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.14 github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect diff --git a/go.sum b/go.sum index 50f54cb..44821e2 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,18 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= +github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= @@ -24,6 +30,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWUR github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/blog/s3.go b/internal/blog/s3.go index de4eb3a..86a6463 100644 --- a/internal/blog/s3.go +++ b/internal/blog/s3.go @@ -11,6 +11,7 @@ import ( "github.com/SayaAndy/saya-today-web/config" "github.com/SayaAndy/saya-today-web/internal/frontmatter" "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" ) @@ -27,12 +28,32 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) { } s3cfg := cfg.Config.(*config.S3Config) - s3cl := s3.New(s3.Options{ - Region: s3cfg.Region, - BaseEndpoint: aws.String(s3cfg.Endpoint), - Credentials: credentials.NewStaticCredentialsProvider(s3cfg.AccessKeyID, s3cfg.SecretAccessKey, ""), + opts := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(s3cfg.Region), + } + if s3cfg.AccessKeyID != "" && s3cfg.SecretAccessKey != "" { + opts = append(opts, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(s3cfg.AccessKeyID, s3cfg.SecretAccessKey, ""), + )) + } + + awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, fmt.Errorf("load AWS config: %w", err) + } + + var s3Opts []func(*s3.Options) + if s3cfg.Endpoint != "" { + s3Opts = append(s3Opts, func(o *s3.Options) { + o.BaseEndpoint = aws.String(s3cfg.Endpoint) + }) + } + s3Opts = append(s3Opts, func(o *s3.Options) { + o.UsePathStyle = s3cfg.UsePathStyle }) + s3cl := s3.NewFromConfig(awsCfg, s3Opts...) + return &S3Client{s3cfg.Prefix, s3cfg.BucketName, s3cl}, nil } -- cgit v1.3.1+13 From ced61eee79d1c06864cd1d580fa6c549365ba213 Mon Sep 17 00:00:00 2001 From: SayaAndy Date: Wed, 15 Apr 2026 16:33:53 +0700 Subject: format: go mod tidy --- go.mod | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'go.mod') diff --git a/go.mod b/go.mod index 373095d..e7a3375 100644 --- a/go.mod +++ b/go.mod @@ -28,10 +28,10 @@ require ( require ( github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.14 - github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect @@ -39,7 +39,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 github.com/aws/smithy-go v1.24.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect -- cgit v1.3.1+13 From ca1f36376865cb2a20923aa62fdff3ee217abde3 Mon Sep 17 00:00:00 2001 From: SayaAndy Date: Sat, 2 May 2026 14:29:55 +0700 Subject: feat: implement rate limiting --- go.mod | 2 ++ go.sum | 4 +++ internal/router/basic-handler.go | 4 +++ internal/router/handlers/api-v1-blog-search.go | 4 +++ .../api-v1-email-send-verification-code.go | 4 +++ internal/router/handlers/api-v1-email-verify.go | 4 +++ internal/router/handlers/api-v1-like-put.go | 4 +++ internal/router/handlers/api-v1-subs-put.go | 4 +++ internal/router/rate-limiters.go | 37 ++++++++++++++++++++++ internal/router/router.go | 5 +++ 10 files changed, 72 insertions(+) create mode 100644 internal/router/rate-limiters.go (limited to 'go.mod') diff --git a/go.mod b/go.mod index e7a3375..c401c7a 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect + github.com/tinylib/msgp v1.2.5 // indirect ) require ( diff --git a/go.sum b/go.sum index 44821e2..d0c2921 100644 --- a/go.sum +++ b/go.sum @@ -91,6 +91,8 @@ github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMux github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.41 h1:8p7Pwz5NHkEbWSqc/ygU4CBGubhFFkpgP9KwcdkAHNA= github.com/mattn/go-sqlite3 v1.14.41/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -100,6 +102,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po= +github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.70.0 h1:LAhMGcWk13QZWm85+eg8ZBNbrq5mnkWFGbHMUJHIdXA= diff --git a/internal/router/basic-handler.go b/internal/router/basic-handler.go index b973d01..68d305c 100644 --- a/internal/router/basic-handler.go +++ b/internal/router/basic-handler.go @@ -42,6 +42,10 @@ func (r *BasicHandler) ContentType() string { return fiber.MIMETextHTMLCharsetUTF8 } +func (r *BasicHandler) RateLimiter() *fiber.Handler { + return nil +} + func (r *BasicHandler) Render(c *fiber.Ctx, supplements *Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) { if !r.IsTemplated() { panic("handler did not implement Render method (while being non-templated)") diff --git a/internal/router/handlers/api-v1-blog-search.go b/internal/router/handlers/api-v1-blog-search.go index c9a91ad..38004a1 100644 --- a/internal/router/handlers/api-v1-blog-search.go +++ b/internal/router/handlers/api-v1-blog-search.go @@ -44,6 +44,10 @@ func (r *BlogSearchHandler) ToValidateLang() router.LangSetting { return router.InForm } +func (r *BlogSearchHandler) RateLimiter() *fiber.Handler { + return &router.RateLimiterLoose +} + func (r *BlogSearchHandler) Render(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) { sort := c.Query("sort") tz := c.Query("tz") diff --git a/internal/router/handlers/api-v1-email-send-verification-code.go b/internal/router/handlers/api-v1-email-send-verification-code.go index 20c5fb6..0b53efa 100644 --- a/internal/router/handlers/api-v1-email-send-verification-code.go +++ b/internal/router/handlers/api-v1-email-send-verification-code.go @@ -38,6 +38,10 @@ func (r *SendVerificationCodeHandler) ToValidateLang() router.LangSetting { return router.InReferer } +func (r *SendVerificationCodeHandler) RateLimiter() *fiber.Handler { + return &router.RateLimiterStrict +} + func (r *SendVerificationCodeHandler) Render(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) { id := c.IP() templateMap["StatusId"] = "email-message" diff --git a/internal/router/handlers/api-v1-email-verify.go b/internal/router/handlers/api-v1-email-verify.go index c9ec969..ace1870 100644 --- a/internal/router/handlers/api-v1-email-verify.go +++ b/internal/router/handlers/api-v1-email-verify.go @@ -36,6 +36,10 @@ func (r *VerifyCodeHandler) ToValidateLang() router.LangSetting { return router.InReferer } +func (r *VerifyCodeHandler) RateLimiter() *fiber.Handler { + return &router.RateLimiterStrict +} + func (r *VerifyCodeHandler) Render(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) { verificationCode := c.FormValue("email_code") templateMap["StatusId"] = "verification-message" diff --git a/internal/router/handlers/api-v1-like-put.go b/internal/router/handlers/api-v1-like-put.go index 434f15a..8305d2b 100644 --- a/internal/router/handlers/api-v1-like-put.go +++ b/internal/router/handlers/api-v1-like-put.go @@ -37,6 +37,10 @@ func (r *PutLikeHandler) ToValidateLang() router.LangSetting { return router.InReferer } +func (r *PutLikeHandler) RateLimiter() *fiber.Handler { + return &router.RateLimiterMedium +} + func (r *PutLikeHandler) Render(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) { path, pathParts, _, err := router.GetPathFromReferer(c) if err != nil { diff --git a/internal/router/handlers/api-v1-subs-put.go b/internal/router/handlers/api-v1-subs-put.go index 144abe5..32fba7e 100644 --- a/internal/router/handlers/api-v1-subs-put.go +++ b/internal/router/handlers/api-v1-subs-put.go @@ -34,6 +34,10 @@ func (r *PutSubsHandler) ToValidateLang() router.LangSetting { return router.InReferer } +func (r *PutSubsHandler) RateLimiter() *fiber.Handler { + return &router.RateLimiterMedium +} + func (r *PutSubsHandler) Render(c *fiber.Ctx, supplements *router.Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) { templateMap["StatusId"] = "subs-message" diff --git a/internal/router/rate-limiters.go b/internal/router/rate-limiters.go new file mode 100644 index 0000000..b43581d --- /dev/null +++ b/internal/router/rate-limiters.go @@ -0,0 +1,37 @@ +package router + +import ( + "time" + + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/middleware/limiter" +) + +var ( + RateLimiterStrict = limiter.New(limiter.Config{ + Max: 5, + Expiration: time.Minute, + KeyGenerator: func(c *fiber.Ctx) string { return c.IP() }, + LimitReached: func(c *fiber.Ctx) error { + return c.Status(fiber.StatusTooManyRequests).SendString("rate limit") + }, + }) + + RateLimiterMedium = limiter.New(limiter.Config{ + Max: 30, + Expiration: time.Minute, + KeyGenerator: func(c *fiber.Ctx) string { return c.IP() }, + LimitReached: func(c *fiber.Ctx) error { + return c.Status(fiber.StatusTooManyRequests).SendString("rate limit") + }, + }) + + RateLimiterLoose = limiter.New(limiter.Config{ + Max: 60, + Expiration: time.Minute, + KeyGenerator: func(c *fiber.Ctx) string { return c.IP() }, + LimitReached: func(c *fiber.Ctx) error { + return c.Status(fiber.StatusTooManyRequests).SendString("rate limit") + }, + }) +) diff --git a/internal/router/router.go b/internal/router/router.go index 926d21a..5230e74 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -78,6 +78,7 @@ type Route interface { TemplatesToInject() []string SitemapInfo(supplements *Supplements) []SitemapInfo ContentType() string + RateLimiter() *fiber.Handler Render(c *fiber.Ctx, supplements *Supplements, lang string, templateMap fiber.Map) (statusCode int, err error) AddMeta(c *fiber.Ctx, supplements *Supplements, lang string, templateMap fiber.Map) (meta []MetaField, err error) AddLinkedData(c *fiber.Ctx, supplements *Supplements, lang string, templateMap fiber.Map) (ld map[string]any, err error) @@ -267,6 +268,10 @@ func (r *Router) InitRoutes() (err error) { return fmt.Errorf("failed to add '%s %s' route into template manager: %w", method, match, err) } + if rateLimiter := route.RateLimiter(); rateLimiter != nil { + r.app.Use(match, *rateLimiter) + } + if route.IsTemplated() { if _, ok := r.templatedRoutes[method]; !ok { r.templatedRoutes[method] = make(map[string]Route) -- cgit v1.3.1+13 From 127e01077493dc58e167716473f051d3ab7d2256 Mon Sep 17 00:00:00 2001 From: Montferrat Date: Sat, 2 May 2026 18:14:28 +0700 Subject: feat: update all go packages --- go.mod | 58 +++++++++++++++++++++++++++++----------------------------- go.sum | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 29 deletions(-) (limited to 'go.mod') diff --git a/go.mod b/go.mod index c401c7a..7d44551 100644 --- a/go.mod +++ b/go.mod @@ -5,44 +5,44 @@ go 1.26 require ( github.com/Backblaze/blazer v0.7.2 github.com/dgraph-io/ristretto/v2 v2.4.0 - github.com/go-co-op/gocron/v2 v2.20.0 + github.com/go-co-op/gocron/v2 v2.21.1 github.com/go-playground/validator/v10 v10.30.2 - github.com/gofiber/fiber/v2 v2.52.12 + github.com/gofiber/fiber/v2 v2.52.13 github.com/golang-migrate/migrate/v4 v4.19.1 - github.com/mattn/go-sqlite3 v1.14.41 + github.com/mattn/go-sqlite3 v1.14.44 github.com/valyala/fasthttp v1.70.0 github.com/wneessen/go-mail v0.7.2 github.com/yuin/goldmark v1.8.2 - golang.org/x/crypto v0.49.0 + golang.org/x/crypto v0.50.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect - github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect - github.com/tinylib/msgp v1.2.5 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect ) require ( github.com/andybalholm/brotli v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/config v1.32.14 - github.com/aws/aws-sdk-go-v2/credentials v1.19.14 - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 - github.com/aws/smithy-go v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 + github.com/aws/smithy-go v1.25.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -51,14 +51,14 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.22 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect ) diff --git a/go.sum b/go.sum index d0c2921..597f315 100644 --- a/go.sum +++ b/go.sum @@ -4,42 +4,78 @@ github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eT github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXqbkiAFvfdyDFwE5c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= @@ -57,6 +93,8 @@ github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9 github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/go-co-op/gocron/v2 v2.20.0 h1:9IMrnnVSWjfSh3E54gWmWCHbloQJLh6f9+nwyKfLNpc= github.com/go-co-op/gocron/v2 v2.20.0/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= +github.com/go-co-op/gocron/v2 v2.21.1 h1:QYOK6iOQVCut+jDcs4zRdWRTBHRxRCEeeFi1TnAmgbU= +github.com/go-co-op/gocron/v2 v2.21.1/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -67,6 +105,8 @@ github.com/go-playground/validator/v10 v10.30.2 h1:JiFIMtSSHb2/XBUbWM4i/MpeQm9ZK github.com/go-playground/validator/v10 v10.30.2/go.mod h1:mAf2pIOVXjTEBrwUMGKkCWKKPs9NheYGabeB04txQSc= github.com/gofiber/fiber/v2 v2.52.12 h1:0LdToKclcPOj8PktUdIKo9BUohjjwfnQl42Dhw8/WUw= github.com/gofiber/fiber/v2 v2.52.12/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= +github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk= +github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -75,6 +115,8 @@ github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbd github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -87,12 +129,20 @@ github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHP github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk= github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.41 h1:8p7Pwz5NHkEbWSqc/ygU4CBGubhFFkpgP9KwcdkAHNA= github.com/mattn/go-sqlite3 v1.14.41/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8= +github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -104,6 +154,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po= github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.70.0 h1:LAhMGcWk13QZWm85+eg8ZBNbrq5mnkWFGbHMUJHIdXA= @@ -118,11 +170,17 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -- cgit v1.3.1+13 From 3efa6ed24713658af855c0ca9504f3e089342182 Mon Sep 17 00:00:00 2001 From: SayaAndy Date: Sat, 2 May 2026 22:08:49 +0700 Subject: feat: move from b2 to s3 for blog pages & facts feat: use centralized index.json for blog pages, keep paginator as fallback feat: multithreaded paginator --- config/config.local.yaml | 24 ++++--- config/config.prod.yaml | 24 ++++--- config/config.stage.yaml | 24 ++++--- go.mod | 1 - go.sum | 61 ----------------- internal/blog/index.go | 32 +++++++++ internal/blog/s3.go | 172 ++++++++++++++++++++++++++++++++++++++++------- 7 files changed, 222 insertions(+), 116 deletions(-) create mode 100644 internal/blog/index.go (limited to 'go.mod') diff --git a/config/config.local.yaml b/config/config.local.yaml index 1822fdc..d3e2e33 100644 --- a/config/config.local.yaml +++ b/config/config.local.yaml @@ -5,22 +5,26 @@ endpoint: listenOn: ":3000" blogPages: storage: - type: b2 + type: s3 config: - bucketName: sayana-pages - region: eu-central-003 + bucketName: sayauz-pages + region: kz1 prefix: "stage-" - keyID: "${B2_KEY_ID}" - applicationKey: "${B2_APPLICATION_KEY}" + endpoint: "https://storage.yandexcloud.kz" + usePathStyle: true + accessKeyID: "${S3_ACCESS_KEY_ID}" + secretAccessKey: "${S3_SECRET_ACCESS_KEY}" factGiver: storage: - type: b2 + type: s3 config: - bucketName: sayana-pages - region: eu-central-003 + bucketName: sayauz-pages + region: kz1 prefix: "" - keyID: "${B2_KEY_ID}" - applicationKey: "${B2_APPLICATION_KEY}" + endpoint: "https://storage.yandexcloud.kz" + usePathStyle: true + accessKeyID: "${S3_ACCESS_KEY_ID}" + secretAccessKey: "${S3_SECRET_ACCESS_KEY}" factsFileName: "facts-*.txt" localePath: ./locale/ availableLanguages: diff --git a/config/config.prod.yaml b/config/config.prod.yaml index 2daa3ff..779550a 100644 --- a/config/config.prod.yaml +++ b/config/config.prod.yaml @@ -6,22 +6,26 @@ endpoint: chmod: '0666' blogPages: storage: - type: b2 + type: s3 config: - bucketName: sayana-pages - region: eu-central-003 + bucketName: sayauz-pages + region: kz1 prefix: "prod-" - keyID: "${B2_KEY_ID}" - applicationKey: "${B2_APPLICATION_KEY}" + endpoint: "https://storage.yandexcloud.kz" + usePathStyle: true + accessKeyID: "${S3_ACCESS_KEY_ID}" + secretAccessKey: "${S3_SECRET_ACCESS_KEY}" factGiver: storage: - type: b2 + type: s3 config: - bucketName: sayana-pages - region: eu-central-003 + bucketName: sayauz-pages + region: kz1 prefix: "" - keyID: "${B2_KEY_ID}" - applicationKey: "${B2_APPLICATION_KEY}" + endpoint: "https://storage.yandexcloud.kz" + usePathStyle: true + accessKeyID: "${S3_ACCESS_KEY_ID}" + secretAccessKey: "${S3_SECRET_ACCESS_KEY}" factsFileName: "facts-*.txt" localePath: ./locale/ availableLanguages: diff --git a/config/config.stage.yaml b/config/config.stage.yaml index 151bb4a..c2e337b 100644 --- a/config/config.stage.yaml +++ b/config/config.stage.yaml @@ -6,22 +6,26 @@ endpoint: chmod: '0666' blogPages: storage: - type: b2 + type: s3 config: - bucketName: sayana-pages - region: eu-central-003 + bucketName: sayauz-pages + region: kz1 prefix: "stage-" - keyID: "${B2_KEY_ID}" - applicationKey: "${B2_APPLICATION_KEY}" + endpoint: "https://storage.yandexcloud.kz" + usePathStyle: true + accessKeyID: "${S3_ACCESS_KEY_ID}" + secretAccessKey: "${S3_SECRET_ACCESS_KEY}" factGiver: storage: - type: b2 + type: s3 config: - bucketName: sayana-pages - region: eu-central-003 + bucketName: sayauz-pages + region: kz1 prefix: "" - keyID: "${B2_KEY_ID}" - applicationKey: "${B2_APPLICATION_KEY}" + endpoint: "https://storage.yandexcloud.kz" + usePathStyle: true + accessKeyID: "${S3_ACCESS_KEY_ID}" + secretAccessKey: "${S3_SECRET_ACCESS_KEY}" factsFileName: "facts-*.txt" localePath: ./locale/ availableLanguages: diff --git a/go.mod b/go.mod index 7d44551..779e641 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect diff --git a/go.sum b/go.sum index 597f315..d277091 100644 --- a/go.sum +++ b/go.sum @@ -2,78 +2,40 @@ github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQ github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= -github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= -github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1 h1:mxuT1xE+dI54NW3RkNjP8DUT5HXqbkiAFvfdyDFwE5c= github.com/aws/aws-sdk-go-v2/service/s3 v1.100.1/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -91,8 +53,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= -github.com/go-co-op/gocron/v2 v2.20.0 h1:9IMrnnVSWjfSh3E54gWmWCHbloQJLh6f9+nwyKfLNpc= -github.com/go-co-op/gocron/v2 v2.20.0/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= github.com/go-co-op/gocron/v2 v2.21.1 h1:QYOK6iOQVCut+jDcs4zRdWRTBHRxRCEeeFi1TnAmgbU= github.com/go-co-op/gocron/v2 v2.21.1/go.mod h1:5lEiCKk1oVJV39Zg7/YG10OnaVrDAV5GGR6O0663k6U= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -103,8 +63,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.30.2 h1:JiFIMtSSHb2/XBUbWM4i/MpeQm9ZK2xqPNk8vgvu5JQ= github.com/go-playground/validator/v10 v10.30.2/go.mod h1:mAf2pIOVXjTEBrwUMGKkCWKKPs9NheYGabeB04txQSc= -github.com/gofiber/fiber/v2 v2.52.12 h1:0LdToKclcPOj8PktUdIKo9BUohjjwfnQl42Dhw8/WUw= -github.com/gofiber/fiber/v2 v2.52.12/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/gofiber/fiber/v2 v2.52.13 h1:TOKP64iqC9b5P49VrBW5tHhUOvDyrtJ0xePEfzJbCbk= github.com/gofiber/fiber/v2 v2.52.13/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= @@ -113,8 +71,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -127,20 +83,12 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-runewidth v0.0.22 h1:76lXsPn6FyHtTY+jt2fTTvsMUCZq1k0qwRsAMuxzKAk= -github.com/mattn/go-runewidth v0.0.22/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/mattn/go-sqlite3 v1.14.41 h1:8p7Pwz5NHkEbWSqc/ygU4CBGubhFFkpgP9KwcdkAHNA= -github.com/mattn/go-sqlite3 v1.14.41/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= github.com/mattn/go-sqlite3 v1.14.44 h1:3VSe+xafpbzsLbdr2AWlAZk9yRHiBhTBakioXaCKTF8= github.com/mattn/go-sqlite3 v1.14.44/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= -github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= -github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -152,8 +100,6 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/tinylib/msgp v1.2.5 h1:WeQg1whrXRFiZusidTQqzETkRpGjFjcIhW6uqWH09po= -github.com/tinylib/msgp v1.2.5/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -168,17 +114,10 @@ github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/blog/index.go b/internal/blog/index.go new file mode 100644 index 0000000..861a90e --- /dev/null +++ b/internal/blog/index.go @@ -0,0 +1,32 @@ +package blog + +import "time" + +const IndexFileName = "index.json" + +const IndexSchemaVersion = 1 + +type IndexEntry struct { + Link string `json:"link"` + ModifiedTime time.Time `json:"modifiedTime"` + Title string `json:"title"` + ShortDescription string `json:"shortDescription"` + ActionDate string `json:"actionDate"` + PublishedTime time.Time `json:"publishedTime"` + Thumbnail string `json:"thumbnail"` + Tags []string `json:"tags"` + Geolocation string `json:"geolocation"` + Medley string `json:"medley,omitempty"` + MedleyPart int `json:"medleyPart,omitempty"` +} + +type IndexCategory struct { + GeneratedAt time.Time `json:"generatedAt"` + Pages []IndexEntry `json:"pages"` +} + +type Index struct { + SchemaVersion int `json:"schemaVersion"` + GeneratedAt time.Time `json:"generatedAt"` + Categories map[string]IndexCategory `json:"categories"` +} diff --git a/internal/blog/s3.go b/internal/blog/s3.go index 86a6463..06630ef 100644 --- a/internal/blog/s3.go +++ b/internal/blog/s3.go @@ -2,10 +2,15 @@ package blog import ( "context" + "encoding/json" + "errors" "fmt" "io" + "log/slog" + "net/url" "slices" "strings" + "sync" "time" "github.com/SayaAndy/saya-today-web/config" @@ -14,8 +19,11 @@ import ( awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" ) +const s3ScanConcurrency = 32 + type S3Client struct { prefix string bucketName string @@ -58,79 +66,195 @@ func NewS3Client(cfg *config.StorageConfig) (Client, error) { } func (c *S3Client) Scan(prefix string) ([]*Page, error) { - pages := []*Page{} + pages, err := c.scanFromIndex(prefix) + if err == nil { + return pages, nil + } - fullPrefix := c.prefix + prefix - input := &s3.ListObjectsV2Input{ + var nsk *s3types.NoSuchKey + if !errors.As(err, &nsk) { + return nil, err + } + + slog.Warn("index.json missing, falling back to listing", slog.String("prefix", c.prefix)) + return c.scanByListing(prefix) +} + +func (c *S3Client) scanFromIndex(prefix string) ([]*Page, error) { + out, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(c.bucketName), - Prefix: aws.String(fullPrefix), + Key: aws.String(IndexFileName), + }) + if err != nil { + return nil, fmt.Errorf("get index.json: %w", err) + } + defer out.Body.Close() + + raw, err := io.ReadAll(out.Body) + if err != nil { + return nil, fmt.Errorf("read index.json: %w", err) + } + + var idx Index + if err := json.Unmarshal(raw, &idx); err != nil { + return nil, fmt.Errorf("unmarshal index.json: %w", err) + } + + wantLang := "" + if i := strings.Index(prefix, "/"); i > 0 { + wantLang = prefix[:i] + } + + fullPrefix := c.prefix + prefix + pages := make([]*Page, 0) + for catKey, cat := range idx.Categories { + lang, ok := strings.CutPrefix(catKey, c.prefix) + if !ok { + continue + } + if wantLang != "" && wantLang != lang { + continue + } + for _, e := range cat.Pages { + if !strings.HasPrefix(e.Link, fullPrefix) { + continue + } + linkParts := strings.Split(e.Link, "/") + nameParts := strings.Split(linkParts[len(linkParts)-1], ".") + fileName := strings.Join(nameParts[:len(nameParts)-1], ".") + pages = append(pages, &Page{ + Link: e.Link, + FileName: fileName, + Lang: lang, + ModifiedTime: e.ModifiedTime, + Metadata: &frontmatter.Metadata{ + Title: e.Title, + ShortDescription: e.ShortDescription, + ActionDate: e.ActionDate, + PublishedTime: e.PublishedTime, + Thumbnail: e.Thumbnail, + Tags: e.Tags, + Geolocation: e.Geolocation, + Medley: e.Medley, + MedleyPart: e.MedleyPart, + }, + }) + } } + return pages, nil +} - paginator := s3.NewListObjectsV2Paginator(c.s3cl, input) +func (c *S3Client) scanByListing(prefix string) ([]*Page, error) { + fullPrefix := c.prefix + prefix + + type candidate struct { + key string + lastModified time.Time + } + var candidates []candidate + paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucketName), + Prefix: aws.String(fullPrefix), + }) for paginator.HasMorePages() { output, err := paginator.NextPage(context.Background()) if err != nil { return nil, fmt.Errorf("list S3 objects: %w", err) } - for _, obj := range output.Contents { key := aws.ToString(obj.Key) - if !strings.HasSuffix(key, ".md") { continue } + candidates = append(candidates, candidate{key, aws.ToTime(obj.LastModified)}) + } + } + + pages := make([]*Page, len(candidates)) + sem := make(chan struct{}, s3ScanConcurrency) + var wg sync.WaitGroup + var firstErr error + var errMu sync.Mutex + + for i, cand := range candidates { + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ Bucket: aws.String(c.bucketName), - Key: aws.String(key), + Key: aws.String(cand.key), }) if err != nil { - return nil, fmt.Errorf("head S3 object %s: %w", key, err) + errMu.Lock() + if firstErr == nil { + firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err) + } + errMu.Unlock() + return } if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") { - continue + return } - meta := head.Metadata if meta["title"] == "" { - continue + return } publishedTime, err := time.Parse(time.RFC3339, meta["published-time"]) if err != nil { - return nil, fmt.Errorf("failed to parse published time metadata field: %w", err) + errMu.Lock() + if firstErr == nil { + firstErr = fmt.Errorf("failed to parse published time metadata field: %w", err) + } + errMu.Unlock() + return } - linkParts := strings.Split(key, "/") + linkParts := strings.Split(cand.key, "/") nameParts := strings.Split(linkParts[len(linkParts)-1], ".") fileName := strings.Join(nameParts[:len(nameParts)-1], ".") + lang, _ := strings.CutPrefix(linkParts[0], c.prefix) tags := strings.Split(meta["tags"], ",") slices.Sort(tags) - lang, _ := strings.CutPrefix(linkParts[0], c.prefix) + title, _ := url.QueryUnescape(meta["title"]) + shortDescription, _ := url.QueryUnescape(meta["short-description"]) + thumbnail, _ := url.QueryUnescape(meta["thumbnail"]) - pages = append(pages, &Page{ - Link: key, + pages[i] = &Page{ + Link: cand.key, FileName: fileName, Lang: lang, - ModifiedTime: aws.ToTime(obj.LastModified), + ModifiedTime: cand.lastModified, Metadata: &frontmatter.Metadata{ - Title: meta["title"], - ShortDescription: meta["short-description"], + Title: title, + ShortDescription: shortDescription, ActionDate: meta["action-date"], PublishedTime: publishedTime, - Thumbnail: meta["thumbnail"], + Thumbnail: thumbnail, Tags: tags, Geolocation: meta["geolocation"], }, - }) - } + } + }) } + wg.Wait() - return pages, nil + if firstErr != nil { + return nil, firstErr + } + + out := pages[:0] + for _, p := range pages { + if p != nil { + out = append(out, p) + } + } + return out, nil } func (c *S3Client) ReadAll(path string) ([]byte, error) { -- cgit v1.3.1+13