diff options
| author | 2026-08-01 19:58:57 +0700 | |
|---|---|---|
| committer | 2026-08-01 19:58:57 +0700 | |
| commit | 684d0b6a57d2eb79730ade63baccdc6e59bebc29 (patch) | |
| tree | 956c89c81965b8c974c8497e3719d0b16d332512 | |
| parent | 129e48683f6fc0615b606a737f2afcd50538bb2e (diff) | |
| download | articlator-main.tar.gz articlator-main.zip | |
feat: repurpose metadata parser as article transcoder for saya.uz andmain
telegram
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | CLAUDE.md | 31 | ||||
| -rw-r--r-- | config/config.go | 64 | ||||
| -rw-r--r-- | config/config.json | 20 | ||||
| -rw-r--r-- | go.mod | 1 | ||||
| -rw-r--r-- | go.sum | 2 | ||||
| -rw-r--r-- | internal/draft/parser.go | 179 | ||||
| -rw-r--r-- | internal/draft/parser_test.go | 83 | ||||
| -rw-r--r-- | internal/storage/b2.go | 142 | ||||
| -rw-r--r-- | internal/storage/index.go | 92 | ||||
| -rw-r--r-- | internal/storage/s3.go | 377 | ||||
| -rw-r--r-- | internal/storage/storage_interface.go | 11 | ||||
| -rw-r--r-- | internal/transcoder/format_test.go | 28 | ||||
| -rw-r--r-- | internal/transcoder/markdown.go | 164 | ||||
| -rw-r--r-- | internal/transcoder/sayauz.go | 138 | ||||
| -rw-r--r-- | internal/transcoder/telegram.go | 584 | ||||
| -rw-r--r-- | internal/transcoder/transcoder.go | 19 | ||||
| -rw-r--r-- | main.go | 105 |
18 files changed, 1731 insertions, 313 deletions
@@ -32,3 +32,7 @@ go.work.sum .vscode/ .envrc + +.draft/ +config*.json +!config*.sample.json @@ -4,7 +4,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -Go service that extracts YAML frontmatter metadata from markdown files stored in cloud object storage (Backblaze B2 or AWS S3), then writes that metadata back as object attributes/metadata. Supports draft/prod workflow where draft files (`.draft.md`) are compared against prod (`.md`) and skipped if unchanged. +Go service that reads markdown draft files from a local directory and runs each through an ordered list of **transcoders** that publish it to different targets: + +- **sayauz** — formats the draft (strips `...` separator lines, stamps a stable `publishedTime`, keeps frontmatter + `{Gallery}` blocks + `---` rules) and uploads the formatted `.md` to cloud object storage (B2/S3) with frontmatter as object metadata; then rebuilds the bucket `index.json`. +- **telegram** — splits the body into sections on `...` lines and posts each *new* section to a Telegram channel as text messages + photo galleries (media groups). Posting is append-only, tracked per draft in a `*.telegram-state.json` sidecar. ## Commands @@ -12,27 +15,33 @@ Go service that extracts YAML frontmatter metadata from markdown files stored in # Build go build -o metadata-extractor main.go -# Run (requires B2_KEY_ID and B2_APPLICATION_KEY env vars, see .envrc) +# Run (env vars in .envrc: S3_*/B2_* for storage, TELEGRAM_BOT_TOKEN/TELEGRAM_CHANNEL_ID) go run main.go -c config/config.json -# No tests or linter configured yet +# Tests +go test ./... ``` ## Architecture -Three-layer pipeline: **Config → Storage → Frontmatter parsing** +Pipeline: **Config → scan local drafts → ParseDraft → each Transcoder → Finalize** -- `main.go` — Entry point. Loads config, scans bucket, fans out goroutines (semaphore-bounded) to process each file: read → parse frontmatter → write metadata back as object attributes. -- `config/config.go` — Loads JSON config with `os.ExpandEnv()` for credential injection. Validates via `validator/v10` struct tags. Storage type (`"b2"` or `"s3"`) selects which config struct and client to use. -- `internal/storage/storage_interface.go` — `StorageClient` interface (`Scan`, `GetReader`, `WriteMetadata`, `CompareDraftAndProd`). Factory map registers all backends. -- `internal/storage/b2.go` — Backblaze B2 implementation. Uses SHA1 for draft/prod change tracking. Writes metadata to B2 object `Info` map. -- `internal/storage/s3.go` — AWS S3 implementation (AWS SDK v2). Uses ETag for draft/prod change tracking. Writes metadata as S3 user metadata. Supports custom `Endpoint` + `UsePathStyle` for S3-compatible stores (MinIO, etc). -- `internal/frontmatter/parser.go` — Extracts content between `---` delimiters, unmarshals YAML into `Metadata` struct. +- `main.go` — Entry point. Loads config, builds transcoders, walks `DraftDir` for `*DraftSuffix` files, fans out goroutines (semaphore-bounded) to parse each draft and run every transcoder in order, then calls `Finalize()` on each once. +- `config/config.go` — Loads JSON config with `os.ExpandEnv()` for credential injection. Validates via `validator/v10`. `Transcoders` is an ordered list; each entry's `Type` (`"sayauz"`/`"telegram"`) selects the config struct via `TranscoderConfig.UnmarshalJSON`. `sayauz` embeds a `StorageConfig` (which itself dispatches `"b2"`/`"s3"`). +- `internal/draft/parser.go` — `ParseDraft` reuses `frontmatter.ParseFrontmatter`, then splits the body into `Section`s on `...` lines, each an ordered list of text / `{Gallery}` `Block`s. Gallery photo lines are `file.jpg [| layoutToken]* [| caption]`; layout tokens (`2x`, `\d+x`) are stripped. +- `internal/transcoder/transcoder.go` — `Transcoder` interface (`Name`, `Transcode`, `Finalize`) + factory map. +- `internal/transcoder/sayauz.go` — `formatContent` does the draft→prod transform; uploads to `{Category}/{codename}.md` via the storage client. `publishedTime` is reused from the draft frontmatter or the existing object's metadata, else stamped `now()`. +- `internal/transcoder/telegram.go` — Bot HTTP API via `net/http` (no SDK dep). `sendMessage`/`sendMediaGroup`; galleries >10 photos split into near-equal groups; per-draft state in `StateDir/{codename}.telegram-state.json`. +- `internal/storage/storage_interface.go` — `StorageClient` interface (`Put`, `GetMetadata`, `BuildIndex`). Factory map registers all backends. +- `internal/storage/b2.go` / `s3.go` — B2 (blazer) / S3 (AWS SDK v2) implementations. S3 url-escapes string metadata (and unescapes in `BuildIndex`); B2 stores plain. S3 supports custom `Endpoint` + `UsePathStyle` for S3-compatible stores. `BuildIndex` is S3-only (B2 returns an error). +- `internal/frontmatter/parser.go` — Extracts content between `---` delimiters, unmarshals YAML into `Metadata`. ## Key Conventions - Metadata keys use kebab-case (e.g., `short-description`, `action-date`) — stored in B2 `Info` map or S3 user metadata - Geolocation format: `"{x} {y}"` or `"{x} {y} {areaError}"` — space-separated floats, validated in both storage backends -- Change tracking: B2 uses SHA1 (`metadata-last-update-sha1`), S3 uses ETag (`metadata-last-update-etag`) +- S3 key for a draft = `{Category}/{codename}.md`; `codename` = draft filename minus `DraftSuffix` +- Section separator in drafts is a line that is exactly `...`; `---` is a kept horizontal rule +- Telegram posting is append-only: only sections whose index is absent from the state file are posted - Semantic commit messages: `feat:`, `fix:`, `refactor:` - Structured logging via `log/slog` diff --git a/config/config.go b/config/config.go index 543413f..64d2e76 100644 --- a/config/config.go +++ b/config/config.go @@ -10,17 +10,63 @@ import ( ) type Config struct { - LogLevel slog.Level `json:"LogLevel" validate:"required"` - MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"` - DraftMode DraftModeConfig `json:"DraftMode"` - Storage StorageConfig `json:"Storage" validate:"required"` + LogLevel slog.Level `json:"LogLevel" validate:"required"` + MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"` + DraftDir string `json:"DraftDir" validate:"required"` + DraftSuffix string `json:"DraftSuffix" validate:"required"` + Transcoders []TranscoderConfig `json:"Transcoders" validate:"required,min=1,dive"` } -type DraftModeConfig struct { - Enabled bool `json:"Enabled" validate:"required"` - DraftSuffix string `json:"DraftSuffix" validate:"required_if=Enabled true"` - ProdSuffix string `json:"ProdSuffix" validate:"required_if=Enabled true"` - UpdateAlways bool `json:"UpdateAlways"` +type TranscoderConfig struct { + Type string `json:"Type" validate:"required,oneof=sayauz telegram"` + Config any `json:"Config" validate:"required"` +} + +func (tc *TranscoderConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + tc.Type = tmp.Type + + switch tmp.Type { + case "sayauz": + var sayauzConfig SayauzConfig + if err := json.Unmarshal(tmp.Config, &sayauzConfig); err != nil { + return fmt.Errorf("unmarshal SayauzConfig: %w", err) + } + tc.Config = &sayauzConfig + case "telegram": + var telegramConfig TelegramConfig + if err := json.Unmarshal(tmp.Config, &telegramConfig); err != nil { + return fmt.Errorf("unmarshal TelegramConfig: %w", err) + } + tc.Config = &telegramConfig + default: + return fmt.Errorf("unsupported transcoder type: %s", tmp.Type) + } + + return nil +} + +type SayauzConfig struct { + Category string `json:"Category" validate:"required,min=1"` + Storage StorageConfig `json:"Storage" validate:"required"` +} + +type TelegramConfig struct { + BotToken string `json:"BotToken" validate:"required,min=1"` + ChannelID int `json:"ChannelID" validate:"required,ne=0"` + GroupID int `json:"GroupID"` + ImageBaseURL string `json:"ImageBaseURL" validate:"required,min=1"` + PreviewBaseURL string `json:"PreviewBaseURL"` + PreviewExtension string `json:"PreviewExtension"` + StateDir string `json:"StateDir" validate:"required,min=1"` } type StorageConfig struct { diff --git a/config/config.json b/config/config.json deleted file mode 100644 index 985dd55..0000000 --- a/config/config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "LogLevel": "debug", - "MaxConcurrentJobs": 10, - "DraftMode": { - "Enabled": true, - "DraftSuffix": ".draft.md", - "ProdSuffix": ".md", - "UpdateAlways": false - }, - "Storage": { - "Type": "b2", - "Config": { - "BucketName": "sayana-pages", - "Region": "eu-central-003", - "Prefix": "", - "KeyID": "${B2_KEY_ID}", - "ApplicationKey": "${B2_APPLICATION_KEY}" - } - } -}
\ No newline at end of file @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.14 github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 github.com/go-playground/validator/v10 v10.27.0 + github.com/yuin/goldmark v1.8.2 gopkg.in/yaml.v3 v3.0.1 ) @@ -56,6 +56,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= diff --git a/internal/draft/parser.go b/internal/draft/parser.go new file mode 100644 index 0000000..2761c87 --- /dev/null +++ b/internal/draft/parser.go @@ -0,0 +1,179 @@ +package draft + +import ( + "regexp" + "strings" + + "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" +) + +// layoutTokenRe matches gallery layout hints like "2x", "3x" that are +// meaningful only to the sayauz renderer and must be dropped elsewhere. +var layoutTokenRe = regexp.MustCompile(`^\d+x$`) + +type BlockKind int + +const ( + TextBlock BlockKind = iota + GalleryBlock +) + +// Block is one ordered piece of a section: either a run of markdown text or a +// gallery. Exactly one of Text / Gallery is meaningful depending on Kind. +type Block struct { + Kind BlockKind + Text string + Gallery *Gallery +} + +type Gallery struct { + Timezone string + KeyPrefix string + Photos []Photo +} + +type Photo struct { + Key string // KeyPrefix + filename, e.g. "Sevsk/20241008-093833.jpg" + Caption string +} + +// Section is the content between two "..." separator lines, split into ordered +// text and gallery blocks. +type Section struct { + Index int + Blocks []Block +} + +type Document struct { + SourcePath string + Codename string + Metadata *frontmatter.Metadata + RawContent []byte + Body string + Sections []Section +} + +// ParseDraft parses a draft file: frontmatter metadata, the markdown body, and +// the body split into sections (on "..." lines) of ordered text/gallery blocks. +func ParseDraft(sourcePath, codename string, content []byte) (*Document, error) { + metadata, body, err := frontmatter.ParseFrontmatter(content) + if err != nil { + return nil, err + } + + doc := &Document{ + SourcePath: sourcePath, + Codename: codename, + Metadata: metadata, + RawContent: content, + Body: string(body), + } + doc.Sections = splitSections(doc.Body) + + return doc, nil +} + +func splitSections(body string) []Section { + lines := strings.Split(body, "\n") + + var sections []Section + var chunk []string + + flush := func() { + blocks := parseBlocks(chunk) + chunk = nil + if len(blocks) == 0 { + return + } + sections = append(sections, Section{Index: len(sections), Blocks: blocks}) + } + + for _, line := range lines { + if strings.TrimSpace(line) == "..." { + flush() + continue + } + chunk = append(chunk, line) + } + flush() + + return sections +} + +func parseBlocks(lines []string) []Block { + var blocks []Block + var text []string + + flushText := func() { + joined := strings.TrimSpace(strings.Join(text, "\n")) + text = nil + if joined != "" { + blocks = append(blocks, Block{Kind: TextBlock, Text: joined}) + } + } + + for i := 0; i < len(lines); i++ { + trimmed := strings.TrimSpace(lines[i]) + if strings.HasPrefix(trimmed, "{Gallery:") { + flushText() + gallery, next := parseGallery(lines, i) + if gallery != nil { + blocks = append(blocks, Block{Kind: GalleryBlock, Gallery: gallery}) + } + i = next + continue + } + text = append(text, lines[i]) + } + flushText() + + return blocks +} + +// parseGallery reads a {Gallery:...} block starting at lines[start] and returns +// the parsed gallery plus the index of the closing {/Gallery} line (or the last +// consumed line if unterminated). +func parseGallery(lines []string, start int) (*Gallery, int) { + header := strings.TrimSpace(lines[start]) + header = strings.TrimPrefix(header, "{Gallery:") + header = strings.TrimSuffix(header, "}") + tz, keyPrefix, _ := strings.Cut(header, ":") + + gallery := &Gallery{Timezone: tz, KeyPrefix: keyPrefix} + + i := start + 1 + for ; i < len(lines); i++ { + trimmed := strings.TrimSpace(lines[i]) + if trimmed == "{/Gallery}" { + break + } + if trimmed == "" { + continue + } + gallery.Photos = append(gallery.Photos, parsePhoto(keyPrefix, trimmed)) + } + + return gallery, i +} + +func parsePhoto(keyPrefix, line string) Photo { + parts := strings.Split(line, "|") + for i := range parts { + parts[i] = strings.TrimSpace(parts[i]) + } + + filename := parts[0] + + var captionParts []string + for _, p := range parts[1:] { + if p == "" || layoutTokenRe.MatchString(p) { + continue + } + captionParts = append(captionParts, p) + } + + return Photo{ + Key: keyPrefix + filename, + Caption: strings.Join(captionParts, " "), + } +} diff --git a/internal/draft/parser_test.go b/internal/draft/parser_test.go new file mode 100644 index 0000000..c6d454d --- /dev/null +++ b/internal/draft/parser_test.go @@ -0,0 +1,83 @@ +package draft + +import ( + "os" + "strings" + "testing" +) + +func TestParseDraftSevsk(t *testing.T) { + raw, err := os.ReadFile("../../.draft/sevsk.md.draft") + if err != nil { + t.Fatal(err) + } + + doc, err := ParseDraft("../../.draft/sevsk.md.draft", "sevsk", raw) + if err != nil { + t.Fatal(err) + } + if doc.Metadata == nil || doc.Metadata.Title == "" { + t.Fatal("expected frontmatter metadata") + } + + var galleries int + for _, section := range doc.Sections { + for _, block := range section.Blocks { + if block.Kind != GalleryBlock { + continue + } + galleries++ + for _, p := range block.Gallery.Photos { + if !strings.HasPrefix(p.Key, "Sevsk/20241008-") { + t.Errorf("unexpected photo key %q", p.Key) + } + if strings.Contains(p.Caption, "2x") { + t.Errorf("layout token leaked into caption %q", p.Caption) + } + } + } + } + if galleries == 0 { + t.Fatal("expected at least one gallery") + } + + // First gallery: 4 photos, captions on photos 1 and 3, none on 2; photo 4 had "2x |" => empty caption. + first := firstGallery(doc) + if first == nil || len(first.Photos) != 4 { + t.Fatalf("first gallery: want 4 photos, got %v", first) + } + if first.Timezone != "Europe/Moscow" { + t.Errorf("first gallery tz = %q", first.Timezone) + } + if first.Photos[0].Caption != "Никольская церковь. Действующая" { + t.Errorf("photo[0] caption = %q", first.Photos[0].Caption) + } + if first.Photos[1].Caption != "" { + t.Errorf("photo[1] caption = %q, want empty", first.Photos[1].Caption) + } + if first.Photos[3].Caption != "" { + t.Errorf("photo[3] caption = %q, want empty (only 2x token)", first.Photos[3].Caption) + } + if first.Photos[3].Key != "Sevsk/20241008-094036.jpg" { + t.Errorf("photo[3] key = %q", first.Photos[3].Key) + } +} + +func firstGallery(doc *Document) *Gallery { + for _, section := range doc.Sections { + for _, block := range section.Blocks { + if block.Kind == GalleryBlock { + return block.Gallery + } + } + } + return nil +} + +func TestSplitSectionsSeparator(t *testing.T) { + body := "intro\n...\nmiddle\n...\nend" + sections := splitSections(body) + if len(sections) != 3 { + t.Fatalf("want 3 sections, got %d", len(sections)) + } +} diff --git a/internal/storage/b2.go b/internal/storage/b2.go index b02261f..6697023 100644 --- a/internal/storage/b2.go +++ b/internal/storage/b2.go @@ -3,7 +3,6 @@ package storage import ( "context" "fmt" - "io" "strconv" "strings" "time" @@ -16,13 +15,12 @@ import ( var _ StorageClient = &B2StorageClient{} type B2StorageClient struct { - prefix string - bucket *b2.Bucket - b2cl *b2.Client - draftModeCfg *config.DraftModeConfig + prefix string + bucket *b2.Bucket + b2cl *b2.Client } -func NewB2StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftModeConfig) (StorageClient, error) { +func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) { if cfg.Type != "b2" { return nil, fmt.Errorf("invalid storage type for B2InputClient") } @@ -38,73 +36,25 @@ func NewB2StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftMod return nil, err } - draftModeCfgCopy := *draftModeCfg - - return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, draftModeCfg: &draftModeCfgCopy}, nil + return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil } -func (sc *B2StorageClient) Scan() ([]string, error) { - filePaths := []string{} - - iter := sc.bucket.List(context.Background(), b2.ListPrefix(sc.prefix)) - - for iter.Next() { - obj := iter.Object() - if obj == nil { - return nil, fmt.Errorf("failed to reference object in B2 bucket") - } - - attrs, err := obj.Attrs(context.Background()) - if err != nil { - return nil, fmt.Errorf("get attributes for object: %w", err) - } - - if attrs.Status != b2.Uploaded { - continue - } - - name := obj.Name() - if !strings.HasSuffix(name, ".md") { - continue - } - - if sc.draftModeCfg.Enabled && !strings.HasSuffix(name, sc.draftModeCfg.DraftSuffix) { - continue - } - - filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix)) - } - - if err := iter.Err(); err != nil { - return nil, fmt.Errorf("iterate over B2 objects: %w", err) - } - - return filePaths, nil -} - -func (sc *B2StorageClient) GetReader(path string) (io.ReadCloser, int64, error) { - obj := sc.bucket.Object(sc.prefix + path) +func (sc *B2StorageClient) GetMetadata(key string) (map[string]string, error) { + obj := sc.bucket.Object(sc.prefix + key) if obj == nil { - return nil, 0, fmt.Errorf("failed to reference object in B2 bucket") + return nil, fmt.Errorf("failed to reference object in B2 bucket") } attrs, err := obj.Attrs(context.Background()) if err != nil { - return nil, 0, fmt.Errorf("error getting attributes of an object: %w", err) + if b2.IsNotExist(err) { + return map[string]string{}, nil + } + return nil, fmt.Errorf("get attributes of B2 object %q: %w", sc.prefix+key, err) } - - return obj.NewReader(context.Background()), attrs.Size, nil + return attrs.Info, nil } -func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error { - draft := sc.bucket.Object(sc.prefix + path) - if draft == nil { - return fmt.Errorf("failed to reference draft object in B2 bucket") - } - draftAttrs, err := draft.Attrs(context.Background()) - if err != nil { - return fmt.Errorf("error getting attributes of a draft object: %w", err) - } - +func (sc *B2StorageClient) Put(key string, content []byte, metadata *frontmatter.Metadata) error { geolocationParts := strings.Split(metadata.Geolocation, " ") if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 { return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string") @@ -131,32 +81,19 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta attrs := &b2.Attrs{ ContentType: "text/markdown; charset=utf-8", Info: map[string]string{ - "title": metadata.Title, - "short-description": metadata.ShortDescription, - "action-date": metadata.ActionDate, - "published-time": metadata.PublishedTime.Format(time.RFC3339), - "thumbnail": metadata.Thumbnail, - "tags": strings.Join(metadata.Tags, ","), - "geolocation": metadata.Geolocation, - "medley": medley, - "metadata-last-update-sha1": draftAttrs.SHA1, + "title": metadata.Title, + "short-description": metadata.ShortDescription, + "action-date": metadata.ActionDate, + "published-time": metadata.PublishedTime.Format(time.RFC3339), + "thumbnail": metadata.Thumbnail, + "tags": strings.Join(metadata.Tags, ","), + "geolocation": metadata.Geolocation, + "medley": medley, }} - reader := draft.NewReader(context.Background()) - content := make([]byte, draftAttrs.Size) - if _, err = reader.Read(content); err != nil { - return fmt.Errorf("failed to read a draft object back for writing (required for attribute setting): %w", err) - } - - var prod *b2.Object - if sc.draftModeCfg.Enabled { - prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix - prod = sc.bucket.Object(sc.prefix + prodPath) - if prod == nil { - return fmt.Errorf("failed to reference prod object in B2 bucket") - } - } else { - prod = draft + prod := sc.bucket.Object(sc.prefix + key) + if prod == nil { + return fmt.Errorf("failed to reference prod object in B2 bucket") } writer := prod.NewWriter(context.Background(), b2.WithAttrsOption(attrs)) @@ -168,31 +105,6 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta return nil } -func (sc *B2StorageClient) CompareDraftAndProd(path string) (changed bool) { - draft := sc.bucket.Object(sc.prefix + path) - if draft == nil { - return false - } - - prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix - prod := sc.bucket.Object(sc.prefix + prodPath) - if prod == nil { - return true - } - - draftAttrs, err := draft.Attrs(context.Background()) - if err != nil { - return false - } - prodAttrs, err := prod.Attrs(context.Background()) - if err != nil { - return true - } - - lastUpdateSha1, ok := prodAttrs.Info["metadata-last-update-sha1"] - if !ok { - return true - } - - return draftAttrs.SHA1 != lastUpdateSha1 +func (sc *B2StorageClient) BuildIndex() error { + return fmt.Errorf("BuildIndex not implemented for B2 storage") } diff --git a/internal/storage/index.go b/internal/storage/index.go new file mode 100644 index 0000000..bfeafbe --- /dev/null +++ b/internal/storage/index.go @@ -0,0 +1,92 @@ +package storage + +import ( + "encoding/json" + "fmt" + "time" +) + +const IndexFileName = "index.json" +const MedleysIndexFileName = "medleys.json" + +const IndexSchemaVersion = 2 + +type IndexEntry struct { + Link string `json:"link"` + ModifiedTime time.Time `json:"modifiedTime"` + Title string `json:"title"` + ShortDescription string `json:"shortDescription"` + ActionDate string `json:"actionDate"` + PublishedTime time.Time `json:"publishedTime"` + Thumbnail string `json:"thumbnail"` + Tags []string `json:"tags"` + Geolocation string `json:"geolocation"` + Medley string `json:"medley,omitempty"` + MedleyPart int `json:"medleyPart,omitempty"` +} + +type IndexV2Category struct { + GeneratedAt time.Time `json:"generatedAt"` + Pages map[string]IndexEntry `json:"pages"` +} + +type IndexV1Category struct { + GeneratedAt time.Time `json:"generatedAt"` + Pages []IndexEntry `json:"pages"` +} + +type Index struct { + SchemaVersion int `json:"schemaVersion"` + GeneratedAt time.Time `json:"generatedAt"` + Categories any `json:"categories"` +} + +func (idx *Index) UnmarshalJSON(data []byte) error { + var tmp struct { + SchemaVersion int `json:"schemaVersion"` + GeneratedAt time.Time `json:"generatedAt"` + Categories json.RawMessage `json:"categories"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + idx.SchemaVersion = tmp.SchemaVersion + idx.GeneratedAt = tmp.GeneratedAt + + switch tmp.SchemaVersion { + case 1: + var categories map[string]*IndexV1Category + if err := json.Unmarshal(tmp.Categories, &categories); err != nil { + return fmt.Errorf("unmarshal map[string]*IndexV1Category: %w", err) + } + idx.Categories = &categories + case 2: + var categories map[string]*IndexV2Category + if err := json.Unmarshal(tmp.Categories, &categories); err != nil { + return fmt.Errorf("unmarshal map[string]*IndexV2Category: %w", err) + } + idx.Categories = &categories + default: + return fmt.Errorf("unsupported index version: %d", tmp.SchemaVersion) + } + + return nil +} + +type IndexV1 struct { + SchemaVersion int `json:"schemaVersion"` + GeneratedAt time.Time `json:"generatedAt"` + Categories map[string]*IndexV1Category `json:"categories"` +} + +type MedleyEntry struct { + Codename string `json:"codename"` + Content []string `json:"content"` +} + +type MedleyPageEntry struct { + Codename string `json:"codename"` + Position int `json:"position"` +} diff --git a/internal/storage/s3.go b/internal/storage/s3.go index 5e9bcd2..5f3acc7 100644 --- a/internal/storage/s3.go +++ b/internal/storage/s3.go @@ -3,31 +3,40 @@ package storage import ( "bytes" "context" + "encoding/json" + "errors" "fmt" "io" + "log/slog" + "maps" + "net/url" + "slices" "strconv" "strings" + "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/SayaAndy/saya-today-article-metadata-add/config" "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" ) +const s3IndexConcurrency = 32 + var _ StorageClient = &S3StorageClient{} type S3StorageClient struct { - prefix string - bucket string - client *s3.Client - draftModeCfg *config.DraftModeConfig + prefix string + bucket string + client *s3.Client } -func NewS3StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftModeConfig) (StorageClient, error) { +func NewS3StorageClient(cfg *config.StorageConfig) (StorageClient, error) { if cfg.Type != "s3" { return nil, fmt.Errorf("invalid storage type for S3StorageClient") } @@ -59,166 +68,322 @@ func NewS3StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftMod client := s3.NewFromConfig(awsCfg, s3Opts...) - draftModeCfgCopy := *draftModeCfg - return &S3StorageClient{ - client: client, - bucket: s3cfg.BucketName, - prefix: s3cfg.Prefix, - draftModeCfg: &draftModeCfgCopy, + client: client, + bucket: s3cfg.BucketName, + prefix: s3cfg.Prefix, }, nil } -func (sc *S3StorageClient) Scan() ([]string, error) { - var filePaths []string +func (sc *S3StorageClient) GetMetadata(key string) (map[string]string, error) { + head, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(sc.prefix + key), + }) + if err != nil { + var nsk *s3types.NoSuchKey + var nf *s3types.NotFound + if errors.As(err, &nsk) || errors.As(err, &nf) { + return map[string]string{}, nil + } + return nil, fmt.Errorf("head S3 object %q: %w", sc.prefix+key, err) + } + return head.Metadata, nil +} + +func (sc *S3StorageClient) Put(key string, content []byte, metadata *frontmatter.Metadata) error { + geolocationParts := strings.Split(metadata.Geolocation, " ") + if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 { + return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string") + } + if len(geolocationParts) >= 2 { + if _, err := strconv.ParseFloat(geolocationParts[0], 64); err != nil { + return fmt.Errorf("invalid geolocation parameter, expected float for X: %w", err) + } + if _, err := strconv.ParseFloat(geolocationParts[1], 64); err != nil { + return fmt.Errorf("invalid geolocation parameter, expected float for Y: %w", err) + } + } + if len(geolocationParts) == 3 { + if _, err := strconv.ParseFloat(geolocationParts[2], 64); err != nil { + return fmt.Errorf("invalid geolocation parameter, expected float for area error: %w", err) + } + } + + s3Metadata := map[string]string{ + "title": url.QueryEscape(metadata.Title), + "short-description": url.QueryEscape(metadata.ShortDescription), + "action-date": metadata.ActionDate, + "published-time": metadata.PublishedTime.Format(time.RFC3339), + "thumbnail": url.QueryEscape(metadata.Thumbnail), + "tags": strings.Join(metadata.Tags, ","), + "geolocation": metadata.Geolocation, + } + + targetKey := sc.prefix + key + _, err := sc.client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(targetKey), + Body: bytes.NewReader(content), + ContentType: aws.String("text/markdown; charset=utf-8"), + Metadata: s3Metadata, + }) + if err != nil { + return fmt.Errorf("put S3 object %q: %w", targetKey, err) + } + + return nil +} + +func (sc *S3StorageClient) BuildIndex() error { + type candidate struct { + key string + codename string + lastModified time.Time + } + var candidates []candidate paginator := s3.NewListObjectsV2Paginator(sc.client, &s3.ListObjectsV2Input{ Bucket: aws.String(sc.bucket), Prefix: aws.String(sc.prefix), }) - for paginator.HasMorePages() { page, err := paginator.NextPage(context.Background()) if err != nil { - return nil, fmt.Errorf("list S3 objects: %w", err) + return fmt.Errorf("list S3 objects: %w", err) } - for _, obj := range page.Contents { - name := aws.ToString(obj.Key) - if !strings.HasSuffix(name, ".md") { + key := aws.ToString(obj.Key) + if key == IndexFileName { continue } - - if sc.draftModeCfg.Enabled && !strings.HasSuffix(name, sc.draftModeCfg.DraftSuffix) { + if !strings.HasSuffix(key, ".md") { continue } + codename := key[strings.LastIndex(key, "/")+1 : strings.LastIndex(key, ".")] + candidates = append(candidates, candidate{key, codename, aws.ToTime(obj.LastModified)}) + } + } - filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix)) + medleys, err := sc.scanMedleys() + if err != nil { + slog.Warn("skipped reading medleys due to an error", slog.String("error", err.Error())) + } + pageToMedleyMap := make(map[string]MedleyPageEntry) + for _, medley := range medleys { + for i, page := range medley.Content { + pageToMedleyMap[page] = MedleyPageEntry{medley.Codename, i} } } - return filePaths, nil -} + type result struct { + catKey string + codename string + entry IndexEntry + } + results := make([]*result, len(candidates)) + sem := make(chan struct{}, s3IndexConcurrency) + var wg sync.WaitGroup + var firstErr error + var errMu sync.Mutex -func (sc *S3StorageClient) GetReader(path string) (io.ReadCloser, int64, error) { - key := sc.prefix + path + for i, cand := range candidates { + sem <- struct{}{} + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-sem }() - out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String(sc.bucket), - Key: aws.String(key), - }) - if err != nil { - return nil, 0, fmt.Errorf("get S3 object %q: %w", key, err) - } + head, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(cand.key), + }) + if err != nil { + errMu.Lock() + if firstErr == nil { + firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err) + } + errMu.Unlock() + return + } - return out.Body, aws.ToInt64(out.ContentLength), nil -} + if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") { + return + } + meta := head.Metadata + if meta["title"] == "" { + return + } -func (sc *S3StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error { - draftKey := sc.prefix + path + publishedTime, err := time.Parse(time.RFC3339, meta["published-time"]) + if err != nil { + slog.Warn("skip entry with bad published-time", slog.String("key", cand.key), slog.String("error", err.Error())) + return + } - getOut, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String(sc.bucket), - Key: aws.String(draftKey), - }) - if err != nil { - return fmt.Errorf("get draft object %q: %w", draftKey, err) - } - content, err := io.ReadAll(getOut.Body) - getOut.Body.Close() - if err != nil { - return fmt.Errorf("read draft object %q: %w", draftKey, err) - } + catKey := cand.key[:strings.Index(cand.key, "/")] - headOut, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ - Bucket: aws.String(sc.bucket), - Key: aws.String(draftKey), - }) - if err != nil { - return fmt.Errorf("head draft object %q: %w", draftKey, err) + tags := strings.Split(meta["tags"], ",") + slices.Sort(tags) + + title, _ := url.QueryUnescape(meta["title"]) + shortDescription, _ := url.QueryUnescape(meta["short-description"]) + thumbnail, _ := url.QueryUnescape(meta["thumbnail"]) + + medleyName, medleyPart := "", 0 + if medley, ok := pageToMedleyMap[cand.codename]; ok { + medleyName, medleyPart = medley.Codename, medley.Position + } + + results[i] = &result{ + catKey: catKey, + codename: cand.codename, + entry: IndexEntry{ + Link: cand.key, + ModifiedTime: cand.lastModified, + Title: title, + ShortDescription: shortDescription, + ActionDate: meta["action-date"], + PublishedTime: publishedTime, + Thumbnail: thumbnail, + Tags: tags, + Geolocation: meta["geolocation"], + Medley: medleyName, + MedleyPart: medleyPart, + }, + } + }() } - draftETag := aws.ToString(headOut.ETag) + wg.Wait() - geolocationParts := strings.Split(metadata.Geolocation, " ") - if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 { - return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string") + if firstErr != nil { + return firstErr } - if len(geolocationParts) >= 2 { - if _, err := strconv.ParseFloat(geolocationParts[0], 64); err != nil { - return fmt.Errorf("invalid geolocation parameter, expected float for X: %w", err) - } - if _, err := strconv.ParseFloat(geolocationParts[1], 64); err != nil { - return fmt.Errorf("invalid geolocation parameter, expected float for Y: %w", err) + + now := time.Now().UTC() + fresh := make(map[string]*IndexV2Category) + for _, r := range results { + if r == nil { + continue } - } - if len(geolocationParts) == 3 { - if _, err := strconv.ParseFloat(geolocationParts[2], 64); err != nil { - return fmt.Errorf("invalid geolocation parameter, expected float for area error: %w", err) + if _, ok := fresh[r.catKey]; !ok { + fresh[r.catKey] = &IndexV2Category{ + Pages: make(map[string]IndexEntry), + } } + fresh[r.catKey].Pages[r.codename] = r.entry + fresh[r.catKey].GeneratedAt = now } - medley := "" - if metadata.Medley != "" { - medley = fmt.Sprintf("%s %d", metadata.Medley, metadata.MedleyPart) + merged, err := sc.loadAndMergeIndex(fresh) + if err != nil { + return err } - s3Metadata := map[string]string{ - "title": metadata.Title, - "short-description": metadata.ShortDescription, - "action-date": metadata.ActionDate, - "published-time": metadata.PublishedTime.Format(time.RFC3339), - "thumbnail": metadata.Thumbnail, - "tags": strings.Join(metadata.Tags, ","), - "geolocation": metadata.Geolocation, - "medley": medley, - "metadata-last-update-etag": draftETag, + idx := Index{ + SchemaVersion: IndexSchemaVersion, + GeneratedAt: now, + Categories: merged, } - targetKey := draftKey - if sc.draftModeCfg.Enabled { - prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix - targetKey = sc.prefix + prodPath + body, err := json.Marshal(idx) + if err != nil { + return fmt.Errorf("marshal index: %w", err) } _, err = sc.client.PutObject(context.Background(), &s3.PutObjectInput{ Bucket: aws.String(sc.bucket), - Key: aws.String(targetKey), - Body: bytes.NewReader(content), - ContentType: aws.String("text/markdown; charset=utf-8"), - Metadata: s3Metadata, + Key: aws.String(IndexFileName), + Body: bytes.NewReader(body), + ContentType: aws.String("application/json; charset=utf-8"), }) if err != nil { - return fmt.Errorf("put S3 object %q: %w", targetKey, err) + return fmt.Errorf("put index %q: %w", IndexFileName, err) } + totalEntries := 0 + for _, c := range merged { + totalEntries += len(c.Pages) + } + slog.Info("wrote index", + slog.String("key", IndexFileName), + slog.Int("categories", len(merged)), + slog.Int("entries", totalEntries)) return nil } -func (sc *S3StorageClient) CompareDraftAndProd(path string) bool { - draftKey := sc.prefix + path - prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix - prodKey := sc.prefix + prodPath - - draftHead, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ +func (sc *S3StorageClient) scanMedleys() ([]MedleyEntry, error) { + out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(sc.bucket), - Key: aws.String(draftKey), + Key: aws.String(MedleysIndexFileName), }) if err != nil { - return false + return nil, fmt.Errorf("get %s: %w", MedleysIndexFileName, err) + } + defer out.Body.Close() + + raw, err := io.ReadAll(out.Body) + if err != nil { + return nil, fmt.Errorf("read %s: %w", MedleysIndexFileName, err) } - prodHead, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + var entries []MedleyEntry + if err := json.Unmarshal(raw, &entries); err != nil { + return nil, fmt.Errorf("unmarshal %s: %w", MedleysIndexFileName, err) + } + + return entries, nil +} + +func (sc *S3StorageClient) loadAndMergeIndex(fresh map[string]*IndexV2Category) (map[string]*IndexV2Category, error) { + merged := make(map[string]*IndexV2Category) + + out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: aws.String(sc.bucket), - Key: aws.String(prodKey), + Key: aws.String(IndexFileName), }) - if err != nil { - return true + if err == nil { + defer out.Body.Close() + raw, readErr := io.ReadAll(out.Body) + if readErr != nil { + return nil, fmt.Errorf("read existing index: %w", readErr) + } + + var legacyIdx Index + if err := json.Unmarshal(raw, &legacyIdx); err != nil { + return nil, fmt.Errorf("unmarshal existing index: %w", err) + } + + switch legacyIdx.SchemaVersion { + case 1: + for k, v := range *legacyIdx.Categories.(*map[string]*IndexV1Category) { + pages := make(map[string]IndexEntry, len(v.Pages)) + for _, page := range v.Pages { + pages[page.Link[strings.LastIndex(page.Link, "/")+1:strings.LastIndex(page.Link, ".")]] = page + } + merged[k] = &IndexV2Category{ + GeneratedAt: v.GeneratedAt, + Pages: pages, + } + } + case 2: + maps.Copy(merged, *legacyIdx.Categories.(*map[string]*IndexV2Category)) + default: + slog.Warn("unknown index schema, discarding", slog.Int("schema_version", legacyIdx.SchemaVersion)) + } + } else { + var nsk *s3types.NoSuchKey + if !errors.As(err, &nsk) { + return nil, fmt.Errorf("get existing index: %w", err) + } } - lastUpdateETag, ok := prodHead.Metadata["metadata-last-update-etag"] - if !ok { - return true + for k := range merged { + if strings.HasPrefix(k, sc.prefix) { + delete(merged, k) + } } + maps.Copy(merged, fresh) - return aws.ToString(draftHead.ETag) != lastUpdateETag + return merged, nil } diff --git a/internal/storage/storage_interface.go b/internal/storage/storage_interface.go index 7134e30..92a37e2 100644 --- a/internal/storage/storage_interface.go +++ b/internal/storage/storage_interface.go @@ -1,20 +1,17 @@ package storage import ( - "io" - "github.com/SayaAndy/saya-today-article-metadata-add/config" "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" ) type StorageClient interface { - Scan() (paths []string, err error) - GetReader(path string) (reader io.ReadCloser, sz int64, err error) - WriteMetadata(path string, metadata *frontmatter.Metadata) error - CompareDraftAndProd(path string) (changed bool) + Put(key string, content []byte, metadata *frontmatter.Metadata) error + GetMetadata(key string) (metadata map[string]string, err error) + BuildIndex() error } -var NewStorageClientMap = map[string]func(*config.StorageConfig, *config.DraftModeConfig) (StorageClient, error){ +var NewStorageClientMap = map[string]func(*config.StorageConfig) (StorageClient, error){ "b2": NewB2StorageClient, "s3": NewS3StorageClient, } diff --git a/internal/transcoder/format_test.go b/internal/transcoder/format_test.go new file mode 100644 index 0000000..9b25dc6 --- /dev/null +++ b/internal/transcoder/format_test.go @@ -0,0 +1,28 @@ +package transcoder + +import ( + "os" + "testing" + "time" +) + +func TestFormatContentMatchesCommitted(t *testing.T) { + draftRaw, err := os.ReadFile("../../.draft/sevsk.md.draft") + if err != nil { + t.Fatal(err) + } + expected, err := os.ReadFile("../../.draft/sevsk.md") + if err != nil { + t.Fatal(err) + } + + published, err := time.Parse(time.RFC3339, "2025-08-26T11:21:53+07:00") + if err != nil { + t.Fatal(err) + } + + got := formatContent(draftRaw, published, true) + if string(got) != string(expected) { + t.Errorf("formatted output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, expected) + } +} diff --git a/internal/transcoder/markdown.go b/internal/transcoder/markdown.go new file mode 100644 index 0000000..975c3bc --- /dev/null +++ b/internal/transcoder/markdown.go @@ -0,0 +1,164 @@ +package transcoder + +import ( + "fmt" + "strings" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + xast "github.com/yuin/goldmark/extension/ast" + "github.com/yuin/goldmark/text" +) + +// telegramMD parses CommonMark once and is reused across calls. +var telegramMD = goldmark.New(goldmark.WithExtensions(extension.Strikethrough, extension.Linkify)) + +// markdownToTelegramHTML converts CommonMark source into the limited HTML +// subset Telegram accepts (parse_mode=HTML). Telegram supports only +// b/i/u/s/a/code/pre/blockquote — block constructs with no Telegram tag +// (paragraphs, headings, lists) are flattened to text + newlines. Anything +// outside the subset is dropped rather than emitted, so a message is never +// rejected for an unsupported tag. +func markdownToTelegramHTML(src string) string { + source := []byte(src) + doc := telegramMD.Parser().Parse(text.NewReader(source)) + + var b strings.Builder + renderNodes(&b, doc, source) + + // Collapse the runs of blank lines block rendering can leave behind. + out := strings.TrimSpace(b.String()) + for strings.Contains(out, "\n\n\n") { + out = strings.ReplaceAll(out, "\n\n\n", "\n\n") + } + return out +} + +func renderNodes(b *strings.Builder, parent ast.Node, source []byte) { + for n := parent.FirstChild(); n != nil; n = n.NextSibling() { + renderNode(b, n, source) + } +} + +func renderNode(b *strings.Builder, n ast.Node, source []byte) { + switch node := n.(type) { + case *ast.Document: + renderNodes(b, node, source) + + case *ast.Paragraph, *ast.TextBlock: + renderNodes(b, node, source) + b.WriteString("\n\n") + + case *ast.Heading: + // Telegram has no headings; render the line in bold. + b.WriteString("<b>") + renderNodes(b, node, source) + b.WriteString("</b>\n\n") + + case *ast.Blockquote: + b.WriteString("<blockquote>") + renderNodes(b, node, source) + trimTrailingNewlines(b) + b.WriteString("</blockquote>\n\n") + + case *ast.List: + renderList(b, node, source) + b.WriteString("\n") + + case *ast.FencedCodeBlock, *ast.CodeBlock: + b.WriteString("<pre>") + writeRawLines(b, n, source) + b.WriteString("</pre>\n\n") + + case *ast.ThematicBreak: + // horizontal rule — nothing meaningful in a Telegram message + + // --- inline --- + case *ast.Text: + b.WriteString(escapeHTML(string(node.Segment.Value(source)))) + if node.HardLineBreak() || node.SoftLineBreak() { + b.WriteByte('\n') + } + case *ast.String: + b.WriteString(escapeHTML(string(node.Value))) + + case *ast.Emphasis: + tag := "i" + if node.Level == 2 { + tag = "b" + } + fmt.Fprintf(b, "<%s>", tag) + renderNodes(b, node, source) + fmt.Fprintf(b, "</%s>", tag) + + case *xast.Strikethrough: + b.WriteString("<s>") + renderNodes(b, node, source) + b.WriteString("</s>") + + case *ast.CodeSpan: + b.WriteString("<code>") + renderNodes(b, node, source) + b.WriteString("</code>") + + case *ast.Link: + fmt.Fprintf(b, `<a href="%s">`, escapeHTML(string(node.Destination))) + renderNodes(b, node, source) + b.WriteString("</a>") + + case *ast.AutoLink: + url := string(node.URL(source)) + fmt.Fprintf(b, `<a href="%s">%s</a>`, escapeHTML(url), escapeHTML(url)) + + case *ast.Image: + // Images can't render inline in text; keep the alt text only. + renderNodes(b, node, source) + + case *ast.RawHTML, *ast.HTMLBlock: + // Drop raw HTML — it is almost certainly not in Telegram's tag subset. + + default: + // Unknown node: recurse so inline text inside it is not lost. + renderNodes(b, n, source) + } +} + +func renderList(b *strings.Builder, list *ast.List, source []byte) { + i := list.Start + for item := list.FirstChild(); item != nil; item = item.NextSibling() { + if list.IsOrdered() { + fmt.Fprintf(b, "%d. ", i) + i++ + } else { + b.WriteString("• ") + } + renderNodes(b, item, source) + trimTrailingNewlines(b) + b.WriteByte('\n') + } +} + +func writeRawLines(b *strings.Builder, n ast.Node, source []byte) { + lines := n.Lines() + for i := 0; i < lines.Len(); i++ { + seg := lines.At(i) + b.WriteString(escapeHTML(string(seg.Value(source)))) + } +} + +func trimTrailingNewlines(b *strings.Builder) { + s := strings.TrimRight(b.String(), "\n") + b.Reset() + b.WriteString(s) +} + +// escapeHTML escapes the three characters Telegram's HTML parser treats as +// markup. Quotes are escaped too so the value is safe inside an href="...". +func escapeHTML(s string) string { + s = strings.ReplaceAll(s, "&", "&") + s = strings.ReplaceAll(s, "<", "<") + s = strings.ReplaceAll(s, ">", ">") + s = strings.ReplaceAll(s, `"`, """) + return s +} diff --git a/internal/transcoder/sayauz.go b/internal/transcoder/sayauz.go new file mode 100644 index 0000000..ee888e5 --- /dev/null +++ b/internal/transcoder/sayauz.go @@ -0,0 +1,138 @@ +package transcoder + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/SayaAndy/saya-today-article-metadata-add/config" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage" +) + +type SayauzTranscoder struct { + category string + storage storage.StorageClient +} + +func NewSayauzTranscoder(cfg any) (Transcoder, error) { + sayauzCfg, ok := cfg.(*config.SayauzConfig) + if !ok { + return nil, fmt.Errorf("invalid config for sayauz transcoder") + } + + newClient, ok := storage.NewStorageClientMap[sayauzCfg.Storage.Type] + if !ok { + return nil, fmt.Errorf("unsupported storage type %q for sayauz transcoder", sayauzCfg.Storage.Type) + } + storageClient, err := newClient(&sayauzCfg.Storage) + if err != nil { + return nil, fmt.Errorf("init storage client: %w", err) + } + + return &SayauzTranscoder{ + category: sayauzCfg.Category, + storage: storageClient, + }, nil +} + +func (t *SayauzTranscoder) Name() string { return "sayauz" } + +func (t *SayauzTranscoder) Transcode(doc *draft.Document) error { + if doc.Metadata == nil { + return fmt.Errorf("draft %q has no frontmatter metadata", doc.SourcePath) + } + + prodKey := t.category + "/" + doc.Codename + ".md" + + // publishedTime is the first-publish time: reuse an existing value (from the + // draft frontmatter, else from the already-published object), otherwise stamp now. + inject := doc.Metadata.PublishedTime.IsZero() + if inject { + published := time.Time{} + if meta, err := t.storage.GetMetadata(prodKey); err == nil { + if v := meta["published-time"]; v != "" { + if parsed, err := time.Parse(time.RFC3339, v); err == nil { + published = parsed + } + } + } else { + slog.Warn("could not read existing metadata for publishedTime, stamping now", + slog.String("key", prodKey), slog.String("error", err.Error())) + } + if published.IsZero() { + published = time.Now() + } + doc.Metadata.PublishedTime = published + } + + formatted := formatContent(doc.RawContent, doc.Metadata.PublishedTime, inject) + + localPath := filepath.Join(filepath.Dir(doc.SourcePath), doc.Codename+".md") + if err := os.WriteFile(localPath, formatted, 0o644); err != nil { + return fmt.Errorf("write formatted file %q: %w", localPath, err) + } + + if err := t.storage.Put(prodKey, formatted, doc.Metadata); err != nil { + return fmt.Errorf("upload %q: %w", prodKey, err) + } + + slog.Info("sayauz published page", + slog.String("key", prodKey), + slog.String("local", localPath)) + return nil +} + +func (t *SayauzTranscoder) Finalize() error { + return t.storage.BuildIndex() +} + +// formatContent renders a draft into its published form: standalone "..." +// separator lines become blank lines (trailing ones are dropped), and when +// inject is true a "publishedTime:" line is added after "actionDate:" in the +// frontmatter. Frontmatter, galleries and "---" rules are otherwise untouched. +func formatContent(raw []byte, published time.Time, inject bool) []byte { + lines := strings.Split(string(raw), "\n") + out := make([]string, 0, len(lines)+1) + + inFrontmatter := false + frontmatterDone := false + + for i, line := range lines { + trimmed := strings.TrimSpace(line) + + if !frontmatterDone { + if i == 0 && trimmed == "---" { + inFrontmatter = true + out = append(out, line) + continue + } + if inFrontmatter { + out = append(out, line) + switch { + case trimmed == "---": + inFrontmatter = false + frontmatterDone = true + case inject && strings.HasPrefix(trimmed, "actionDate:"): + out = append(out, "publishedTime: "+published.Format(time.RFC3339)) + } + continue + } + } + + if trimmed == "..." { + out = append(out, "") + continue + } + out = append(out, line) + } + + for len(out) > 0 && out[len(out)-1] == "" { + out = out[:len(out)-1] + } + + return []byte(strings.Join(out, "\n")) +} diff --git a/internal/transcoder/telegram.go b/internal/transcoder/telegram.go new file mode 100644 index 0000000..b521fb0 --- /dev/null +++ b/internal/transcoder/telegram.go @@ -0,0 +1,584 @@ +package transcoder + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/SayaAndy/saya-today-article-metadata-add/config" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft" +) + +const ( + telegramMediaGroupLimit = 10 + // telegramCaptionLimit is the max length (runes) of a media caption. + telegramCaptionLimit = 1024 + // telegramMaxAttempts caps retries when the API returns 429. + telegramMaxAttempts = 5 +) + +type TelegramTranscoder struct { + botToken string + channelID int + groupID int + lastUpdateID int + imageBaseURL string + previewBaseURL string + previewExtension string + stateDir string + http *http.Client +} + +func NewTelegramTranscoder(cfg any) (Transcoder, error) { + tgCfg, ok := cfg.(*config.TelegramConfig) + if !ok { + return nil, fmt.Errorf("invalid config for telegram transcoder") + } + + transcoder := &TelegramTranscoder{ + botToken: tgCfg.BotToken, + channelID: tgCfg.ChannelID, + groupID: tgCfg.GroupID, + lastUpdateID: -1, + imageBaseURL: strings.TrimRight(tgCfg.ImageBaseURL, "/"), + previewBaseURL: strings.TrimRight(tgCfg.PreviewBaseURL, "/"), + previewExtension: tgCfg.PreviewExtension, + stateDir: tgCfg.StateDir, + http: &http.Client{Timeout: time.Minute}, + } + + if _, err := transcoder.getUpdates(); err != nil { + return nil, fmt.Errorf("failed to get last update id for checkpoint: %w", err) + } + + return transcoder, nil +} + +func (t *TelegramTranscoder) Name() string { return "telegram" } + +func (t *TelegramTranscoder) Finalize() error { return nil } + +// telegramState records, per draft, which section indices have already been +// posted and the message ids they produced. Posting is append-only: a section +// already present here is never re-sent. +type telegramState struct { + ChannelID string `json:"channelID"` + Posted map[string][]int `json:"posted"` +} + +func (t *TelegramTranscoder) Transcode(doc *draft.Document) error { + statePath := filepath.Join(t.stateDir, doc.Codename+".telegram-state.json") + state, err := loadTelegramState(statePath) + if err != nil { + return fmt.Errorf("load telegram state %q: %w", statePath, err) + } + + if state.ChannelID != "" && state.ChannelID != strconv.Itoa(t.channelID) { + slog.Warn("telegram channel changed, re-posting all sections to the new channel", + slog.String("codename", doc.Codename), + slog.String("old", state.ChannelID), + slog.Int("new", t.channelID)) + state.Posted = map[string][]int{} + } + state.ChannelID = strconv.Itoa(t.channelID) + + for _, section := range doc.Sections { + idx := strconv.Itoa(section.Index) + if _, done := state.Posted[idx]; done { + continue + } + + var msgIDs []int + pendingText := "" + + // flushText sends any buffered text as its own message. + flushText := func() error { + if pendingText == "" { + return nil + } + id, err := t.sendMessage(pendingText) + if err != nil { + return err + } + msgIDs = append(msgIDs, id) + pendingText = "" + return nil + } + + for _, block := range section.Blocks { + switch block.Kind { + case draft.TextBlock: + text := stripRules(block.Text) + if text == "" { + continue + } + // flush any earlier text before buffering this one + if err := flushText(); err != nil { + return fmt.Errorf("send section %d text: %w", section.Index, err) + } + pendingText = text + case draft.GalleryBlock: + // Attach the immediately preceding text as the album caption when + // it fits; otherwise send it as its own message first. + caption := "" + if pendingText != "" { + if len([]rune(pendingText)) <= telegramCaptionLimit { + caption = pendingText + pendingText = "" + } else if err := flushText(); err != nil { + return fmt.Errorf("send section %d text: %w", section.Index, err) + } + } + for i, group := range splitPhotos(block.Gallery.Photos, telegramMediaGroupLimit) { + groupCaption := "" + if i == 0 { + groupCaption = caption + } + ids, err := t.sendMediaGroup(group, groupCaption, "photo", t.channelID, 0) + if err != nil { + return fmt.Errorf("send section %d gallery: %w", section.Index, err) + } + msgIDs = append(msgIDs, ids...) + if err := flushText(); err != nil { + return fmt.Errorf("send section %d text: %w", section.Index, err) + } + if len(msgIDs) > 0 { + time.Sleep(5 * time.Second) + forwardedID, err := t.getForwardedID(msgIDs) + if err != nil { + return fmt.Errorf("send full documents for section %d gallery: %w", section.Index, err) + } + if forwardedID == 0 { + return fmt.Errorf("fail to get forwarded id of section %d gallery", section.Index) + } + _, err = t.sendMediaGroup(group, "", "document", t.groupID, forwardedID) + if err != nil { + return fmt.Errorf("send section %d documents: %w", section.Index, err) + } + } + } + } + } + + state.Posted[idx] = msgIDs + if err := saveTelegramState(statePath, state); err != nil { + return fmt.Errorf("save telegram state %q: %w", statePath, err) + } + slog.Info("telegram posted section", + slog.String("codename", doc.Codename), + slog.Int("section", section.Index), + slog.Int("messages", len(msgIDs))) + } + + return nil +} + +func loadTelegramState(path string) (*telegramState, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return &telegramState{Posted: map[string][]int{}}, nil + } + return nil, err + } + var state telegramState + if err := json.Unmarshal(raw, &state); err != nil { + return nil, err + } + if state.Posted == nil { + state.Posted = map[string][]int{} + } + return &state, nil +} + +func saveTelegramState(path string, state *telegramState) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + raw, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, raw, 0o644) +} + +// stripRules drops standalone "---" horizontal-rule lines (meaningless in a +// Telegram message) and trims the remaining text. +func stripRules(text string) string { + lines := strings.Split(text, "\n") + kept := lines[:0] + for _, line := range lines { + if strings.TrimSpace(line) == "---" { + continue + } + kept = append(kept, line) + } + return strings.TrimSpace(strings.Join(kept, "\n")) +} + +// splitPhotos divides photos into near-equal groups each no larger than limit. +func splitPhotos(photos []draft.Photo, limit int) [][]draft.Photo { + n := len(photos) + if n == 0 { + return nil + } + groupCount := (n + limit - 1) / limit + base := n / groupCount + rem := n % groupCount + + var groups [][]draft.Photo + start := 0 + for g := range groupCount { + size := base + if g < rem { + size++ + } + groups = append(groups, photos[start:start+size]) + start += size + } + return groups +} + +type inputMediaPhoto struct { + Type string `json:"type"` + Media string `json:"media"` + Caption string `json:"caption,omitempty"` + ParseMode string `json:"parse_mode,omitempty"` +} + +func (t *TelegramTranscoder) photoURL(key string) string { + return t.imageBaseURL + "/" + key +} + +func (t *TelegramTranscoder) previewURL(key string) string { + base := t.previewBaseURL + if base == "" { + base = t.imageBaseURL + } + if t.previewExtension != "" { + key = strings.TrimSuffix(key, filepath.Ext(key)) + t.previewExtension + } + return base + "/" + key +} + +func (t *TelegramTranscoder) sendMessage(text string) (int, error) { + payload := map[string]any{ + "chat_id": strconv.Itoa(t.channelID), + "text": markdownToTelegramHTML(text), + "parse_mode": "HTML", + "disable_web_page_preview": true, + } + var result struct { + MessageID int `json:"message_id"` + } + if err := t.call("sendMessage", payload, &result); err != nil { + return 0, err + } + return result.MessageID, nil +} + +func (t *TelegramTranscoder) sendMediaGroup(photos []draft.Photo, caption string, sendAsType string, id int, replyTo int) ([]int, error) { + media := make([]inputMediaPhoto, 0, len(photos)) + files := map[string]uploadFile{} + for i, p := range photos { + item := inputMediaPhoto{Type: sendAsType} + if sendAsType == "document" { + // Upload the original file directly rather than passing Telegram a + // URL to fetch: full-resolution originals routinely fail Telegram's + // server-side downloader with WEBPAGE_CURL_FAILED. + data, err := t.downloadPhoto(p.Key) + if err != nil { + return nil, fmt.Errorf("download %q for upload: %w", p.Key, err) + } + field := fmt.Sprintf("file%d", i) + files[field] = uploadFile{name: filepath.Base(p.Key), data: data} + item.Media = "attach://" + field + } else { + item.Media = t.previewURL(p.Key) + } + // Telegram shows an album's caption in the feed only when exactly one + // item is captioned. So the section text goes on the first photo and + // every other photo is left uncaptioned (per-photo captions dropped). + if i == 0 && caption != "" { + item.Caption = markdownToTelegramHTML(caption) + item.ParseMode = "HTML" + } + media = append(media, item) + } + + var result []struct { + MessageID int `json:"message_id"` + } + + if len(files) > 0 { + mediaJSON, err := json.Marshal(media) + if err != nil { + return nil, err + } + fields := map[string]string{ + "chat_id": strconv.Itoa(id), + "media": string(mediaJSON), + } + if replyTo != 0 { + replyJSON, err := json.Marshal(map[string]any{"message_id": replyTo}) + if err != nil { + return nil, err + } + fields["reply_parameters"] = string(replyJSON) + } + if err := t.callMultipart("sendMediaGroup", fields, files, &result); err != nil { + return nil, err + } + } else { + payload := map[string]any{ + "chat_id": strconv.Itoa(id), + "media": media, + } + if replyTo != 0 { + payload["reply_parameters"] = map[string]any{ + "message_id": replyTo, + } + } + if err := t.call("sendMediaGroup", payload, &result); err != nil { + return nil, err + } + } + + ids := make([]int, 0, len(result)) + for _, r := range result { + ids = append(ids, r.MessageID) + } + return ids, nil +} + +type uploadFile struct { + name string + data []byte +} + +// downloadPhoto fetches the original bytes for a photo key so they can be +// uploaded directly to Telegram instead of fetched server-side by Telegram. +func (t *TelegramTranscoder) downloadPhoto(key string) ([]byte, error) { + url := t.photoURL(key) + resp, err := t.http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %d fetching %q", resp.StatusCode, url) + } + return io.ReadAll(resp.Body) +} + +func (t *TelegramTranscoder) getForwardedID(channelMessageIDs []int) (int, error) { + var payloads []getUpdatesPayload + var err error + slog.Debug("search for forwarded message id", slog.Any("original_id", channelMessageIDs), slog.Int("group_id", t.groupID), slog.Int("channel_id", t.channelID)) + + for range 3 { + for payloads, err = t.getUpdates(); len(payloads) > 0 && err == nil; payloads, err = t.getUpdates() { + for _, payload := range payloads { + if payload.Message.MessageOrigin.Type == "channel" && + payload.Message.MessageOrigin.Chat.ID == t.channelID && + slices.Contains(channelMessageIDs, payload.Message.MessageOrigin.MessageID) && + payload.Message.Chat.ID == t.groupID { + return payload.Message.MessageID, nil + } + } + } + time.Sleep(3 * time.Second) + } + + return 0, err +} + +type getUpdatesPayload struct { + UpdateID int `json:"update_id"` + Message struct { + MessageID int `json:"message_id"` + Chat struct { + ID int `json:"id"` + Title string `json:"title"` + Type string `json:"type"` + } `json:"chat"` + MessageOrigin struct { + Type string `json:"type"` + Date int `json:"date"` + Chat struct { + ID int `json:"id"` + Title string `json:"title"` + Type string `json:"type"` + } `json:"chat"` + MessageID int `json:"message_id"` + } `json:"forward_origin"` + Date int `json:"date"` + Text string `json:"text"` + } `json:"message"` +} + +func (t *TelegramTranscoder) getUpdates() ([]getUpdatesPayload, error) { + var out []getUpdatesPayload + + payload := struct { + Offset int `json:"offset"` + }{ + Offset: t.lastUpdateID, + } + + if err := t.call("getUpdates", payload, &out); err != nil { + return nil, fmt.Errorf("failed to transcode received message: %w", err) + } + + for _, p := range out { + pjson, _ := json.Marshal(p) + slog.Debug("parsed new update", slog.String("payload", string(pjson))) + if p.UpdateID > t.lastUpdateID { + t.lastUpdateID = p.UpdateID + } + } + t.lastUpdateID++ + + return out, nil +} + +// handleResponse decodes a Telegram API response envelope. When the API asks us +// to back off (HTTP 429) it returns a positive wait duration and a nil error so +// the caller can retry; otherwise it unmarshals the result into out (if any) or +// returns the API error verbatim. +func (t *TelegramTranscoder) handleResponse(method string, resp *http.Response, attempt int, out any) (time.Duration, error) { + var envelope struct { + OK bool `json:"ok"` + ErrorCode int `json:"error_code"` + Description string `json:"description"` + Parameters struct { + RetryAfter int `json:"retry_after"` + } `json:"parameters"` + Result json.RawMessage `json:"result"` + } + decodeErr := json.NewDecoder(resp.Body).Decode(&envelope) + resp.Body.Close() + if decodeErr != nil { + return 0, fmt.Errorf("decode telegram response (%s): %w", method, decodeErr) + } + + envelopeBytes, _ := json.Marshal(envelope) + slog.Debug("telegram response", slog.String("method", method), slog.String("response", string(envelopeBytes))) + + if envelope.OK { + if out == nil { + return 0, nil + } + return 0, json.Unmarshal(envelope.Result, out) + } + + // 429 Too Many Requests: Telegram tells us how long to wait in + // parameters.retry_after. Back off and retry instead of failing. + if envelope.ErrorCode == 429 && attempt < telegramMaxAttempts { + wait := time.Duration(envelope.Parameters.RetryAfter) * time.Second + if wait <= 0 { + wait = time.Second + } + slog.Warn("telegram rate limited, backing off", + slog.String("method", method), + slog.Int("attempt", attempt), + slog.Duration("retry_after", wait)) + return wait, nil + } + + return 0, fmt.Errorf("telegram %s failed: %s", method, envelope.Description) +} + +// call invokes a Telegram Bot API method with a JSON body and decodes the +// "result" field into out. The API's error description is surfaced verbatim. +func (t *TelegramTranscoder) call(method string, payload any, out any) error { + body, err := json.Marshal(payload) + if err != nil { + return err + } + + url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", t.botToken, method) + + for attempt := 1; ; attempt++ { + req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + slog.Debug("send message", slog.String("method", method), slog.String("request", string(body))) + + resp, err := t.http.Do(req) + if err != nil { + return err + } + + wait, err := t.handleResponse(method, resp, attempt, out) + if wait > 0 { + time.Sleep(wait) + continue + } + return err + } +} + +// callMultipart invokes a Telegram Bot API method using multipart/form-data, +// uploading the given files directly instead of handing Telegram URLs to fetch +// server-side. fields carries the non-file form values (chat_id, media JSON, +// reply_parameters, ...); files maps a form field name to its bytes and upload +// filename, referenced from the media JSON via attach://<field>. +func (t *TelegramTranscoder) callMultipart(method string, fields map[string]string, files map[string]uploadFile, out any) error { + url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", t.botToken, method) + + for attempt := 1; ; attempt++ { + var body bytes.Buffer + w := multipart.NewWriter(&body) + for k, v := range fields { + if err := w.WriteField(k, v); err != nil { + return err + } + } + for field, f := range files { + part, err := w.CreateFormFile(field, f.name) + if err != nil { + return err + } + if _, err := part.Write(f.data); err != nil { + return err + } + } + if err := w.Close(); err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, url, &body) + if err != nil { + return err + } + req.Header.Set("Content-Type", w.FormDataContentType()) + + slog.Debug("send media group upload", slog.String("method", method), slog.Int("files", len(files))) + + resp, err := t.http.Do(req) + if err != nil { + return err + } + + wait, err := t.handleResponse(method, resp, attempt, out) + if wait > 0 { + time.Sleep(wait) + continue + } + return err + } +} diff --git a/internal/transcoder/transcoder.go b/internal/transcoder/transcoder.go new file mode 100644 index 0000000..3dbdf7e --- /dev/null +++ b/internal/transcoder/transcoder.go @@ -0,0 +1,19 @@ +package transcoder + +import ( + "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft" +) + +// Transcoder turns a parsed draft into a published artifact (an S3 page, a +// Telegram thread, ...). Transcode runs once per draft; Finalize runs once +// after all drafts have been processed. +type Transcoder interface { + Name() string + Transcode(doc *draft.Document) error + Finalize() error +} + +var NewTranscoderMap = map[string]func(cfg any) (Transcoder, error){ + "sayauz": NewSayauzTranscoder, + "telegram": NewTelegramTranscoder, +} @@ -2,13 +2,16 @@ package main import ( "flag" + "io/fs" "log/slog" "os" + "path/filepath" + "strings" "sync" "github.com/SayaAndy/saya-today-article-metadata-add/config" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/transcoder" ) var configPath = flag.String("c", "config.json", "Path to the configuration file") @@ -16,8 +19,8 @@ var configPath = flag.String("c", "config.json", "Path to the configuration file func main() { flag.Parse() - cfg := &config.Config{} - if err := config.LoadConfig(*configPath, cfg); err != nil { + cfg, err := config.InitConfig(*configPath) + if err != nil { slog.Error("fail to load configuration", slog.String("error", err.Error())) os.Exit(1) } @@ -25,73 +28,85 @@ func main() { slog.SetLogLoggerLevel(cfg.LogLevel) slog.Info("starting metadata extractor...") - storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage, &cfg.DraftMode) - if err != nil { - slog.Error("fail to initialize input client", slog.String("error", err.Error())) - os.Exit(1) + transcoders := make([]transcoder.Transcoder, 0, len(cfg.Transcoders)) + for _, tcCfg := range cfg.Transcoders { + newTranscoder, ok := transcoder.NewTranscoderMap[tcCfg.Type] + if !ok { + slog.Error("unsupported transcoder type", slog.String("type", tcCfg.Type)) + os.Exit(1) + } + t, err := newTranscoder(tcCfg.Config) + if err != nil { + slog.Error("fail to initialize transcoder", slog.String("type", tcCfg.Type), slog.String("error", err.Error())) + os.Exit(1) + } + transcoders = append(transcoders, t) + slog.Info("initialized transcoder", slog.String("type", tcCfg.Type)) } - generalLogger := slog.With( - slog.String("storage_type", cfg.Storage.Type), - ) - generalLogger.Info("initialized storage client") - - files, err := storageClient.Scan() + var drafts []string + err = filepath.WalkDir(cfg.DraftDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(path, cfg.DraftSuffix) { + drafts = append(drafts, path) + } + return nil + }) if err != nil { - generalLogger.Error("fail to scan input files", slog.String("error", err.Error())) + slog.Error("fail to scan draft directory", slog.String("dir", cfg.DraftDir), slog.String("error", err.Error())) os.Exit(1) } - generalLogger.Info("scanned files", slog.Int("file_count", len(files))) + slog.Info("scanned drafts", slog.Int("draft_count", len(drafts))) semaphore := make(chan struct{}, cfg.MaxConcurrentJobs) var wg sync.WaitGroup - wg.Add(len(files)) + wg.Add(len(drafts)) - for i, file := range files { + for _, draftPath := range drafts { semaphore <- struct{}{} - go func(index int, inputName string) { + go func(path string) { defer wg.Done() defer func() { <-semaphore }() - if cfg.DraftMode.Enabled && !storageClient.CompareDraftAndProd(inputName) { - generalLogger.Debug("skip a draft because prod object is identical to it", slog.String("file", inputName)) - return - } - generalLogger.Debug("processing a file", slog.String("file", inputName)) - reader, sz, err := storageClient.GetReader(inputName) - if err != nil { - generalLogger.Warn("fail to get reader for a file", slog.String("file", inputName), slog.String("error", err.Error())) - return - } - defer reader.Close() + codename := strings.TrimSuffix(filepath.Base(path), cfg.DraftSuffix) + fileLogger := slog.With(slog.String("draft", path), slog.String("codename", codename)) - content := make([]byte, sz) - ln, err := reader.Read(content) + content, err := os.ReadFile(path) if err != nil { - generalLogger.Warn("fail to read content from a file", slog.String("file", inputName), slog.String("error", err.Error())) + fileLogger.Warn("fail to read draft", slog.String("error", err.Error())) return } - generalLogger.Debug("read content from a file", - slog.String("file", inputName), - slog.Int64("expected_size", sz), - slog.Int("output_size", ln)) - metadata, _, err := frontmatter.ParseFrontmatter(content) + doc, err := draft.ParseDraft(path, codename, content) if err != nil { - generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", inputName), slog.String("error", err.Error())) + fileLogger.Warn("fail to parse draft", slog.String("error", err.Error())) return } - - if metadata == nil { - generalLogger.Info("skip a file due to it not having metadata", slog.String("file", inputName)) + if doc.Metadata == nil { + fileLogger.Info("skip draft without frontmatter metadata") return } - if err = storageClient.WriteMetadata(inputName, metadata); err != nil { - generalLogger.Warn("fail to write metadata to a file", slog.String("file", inputName), slog.String("error", err.Error())) + for _, t := range transcoders { + if err := t.Transcode(doc); err != nil { + fileLogger.Warn("transcoder failed", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) + } } - }(i, file) + }(draftPath) } wg.Wait() + + for _, t := range transcoders { + if err := t.Finalize(); err != nil { + slog.Error("fail to finalize transcoder", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) + os.Exit(1) + } + slog.Info("finalized transcoder", slog.String("transcoder", t.Name())) + } } |