summaryrefslogtreecommitdiff
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore6
-rw-r--r--CLAUDE.md47
-rw-r--r--config/config.go78
-rw-r--r--config/config.json14
-rw-r--r--go.mod29
-rw-r--r--go.sum49
-rw-r--r--internal/draft/parser.go179
-rw-r--r--internal/draft/parser_test.go83
-rw-r--r--internal/frontmatter/parser.go3
-rw-r--r--internal/storage/b2.go110
-rw-r--r--internal/storage/index.go92
-rw-r--r--internal/storage/s3.go389
-rw-r--r--internal/storage/storage_interface.go12
-rw-r--r--internal/transcoder/format_test.go28
-rw-r--r--internal/transcoder/markdown.go164
-rw-r--r--internal/transcoder/sayauz.go138
-rw-r--r--internal/transcoder/telegram.go584
-rw-r--r--internal/transcoder/transcoder.go19
-rw-r--r--main.go105
19 files changed, 156 insertions, 1973 deletions
diff --git a/.gitignore b/.gitignore
index 46b198b..ed20272 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,9 +30,3 @@ go.work.sum
# Editor/IDE
.idea/
.vscode/
-
-.envrc
-
-.draft/
-config*.json
-!config*.sample.json
diff --git a/CLAUDE.md b/CLAUDE.md
deleted file mode 100644
index 37d4651..0000000
--- a/CLAUDE.md
+++ /dev/null
@@ -1,47 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Project Overview
-
-Go service that reads markdown draft files from a local directory and runs each through an ordered list of **transcoders** that publish it to different targets:
-
-- **sayauz** — formats the draft (strips `...` separator lines, stamps a stable `publishedTime`, keeps frontmatter + `{Gallery}` blocks + `---` rules) and uploads the formatted `.md` to cloud object storage (B2/S3) with frontmatter as object metadata; then rebuilds the bucket `index.json`.
-- **telegram** — splits the body into sections on `...` lines and posts each *new* section to a Telegram channel as text messages + photo galleries (media groups). Posting is append-only, tracked per draft in a `*.telegram-state.json` sidecar.
-
-## Commands
-
-```bash
-# Build
-go build -o metadata-extractor main.go
-
-# Run (env vars in .envrc: S3_*/B2_* for storage, TELEGRAM_BOT_TOKEN/TELEGRAM_CHANNEL_ID)
-go run main.go -c config/config.json
-
-# Tests
-go test ./...
-```
-
-## Architecture
-
-Pipeline: **Config → scan local drafts → ParseDraft → each Transcoder → Finalize**
-
-- `main.go` — Entry point. Loads config, builds transcoders, walks `DraftDir` for `*DraftSuffix` files, fans out goroutines (semaphore-bounded) to parse each draft and run every transcoder in order, then calls `Finalize()` on each once.
-- `config/config.go` — Loads JSON config with `os.ExpandEnv()` for credential injection. Validates via `validator/v10`. `Transcoders` is an ordered list; each entry's `Type` (`"sayauz"`/`"telegram"`) selects the config struct via `TranscoderConfig.UnmarshalJSON`. `sayauz` embeds a `StorageConfig` (which itself dispatches `"b2"`/`"s3"`).
-- `internal/draft/parser.go` — `ParseDraft` reuses `frontmatter.ParseFrontmatter`, then splits the body into `Section`s on `...` lines, each an ordered list of text / `{Gallery}` `Block`s. Gallery photo lines are `file.jpg [| layoutToken]* [| caption]`; layout tokens (`2x`, `\d+x`) are stripped.
-- `internal/transcoder/transcoder.go` — `Transcoder` interface (`Name`, `Transcode`, `Finalize`) + factory map.
-- `internal/transcoder/sayauz.go` — `formatContent` does the draft→prod transform; uploads to `{Category}/{codename}.md` via the storage client. `publishedTime` is reused from the draft frontmatter or the existing object's metadata, else stamped `now()`.
-- `internal/transcoder/telegram.go` — Bot HTTP API via `net/http` (no SDK dep). `sendMessage`/`sendMediaGroup`; galleries >10 photos split into near-equal groups; per-draft state in `StateDir/{codename}.telegram-state.json`.
-- `internal/storage/storage_interface.go` — `StorageClient` interface (`Put`, `GetMetadata`, `BuildIndex`). Factory map registers all backends.
-- `internal/storage/b2.go` / `s3.go` — B2 (blazer) / S3 (AWS SDK v2) implementations. S3 url-escapes string metadata (and unescapes in `BuildIndex`); B2 stores plain. S3 supports custom `Endpoint` + `UsePathStyle` for S3-compatible stores. `BuildIndex` is S3-only (B2 returns an error).
-- `internal/frontmatter/parser.go` — Extracts content between `---` delimiters, unmarshals YAML into `Metadata`.
-
-## Key Conventions
-
-- Metadata keys use kebab-case (e.g., `short-description`, `action-date`) — stored in B2 `Info` map or S3 user metadata
-- Geolocation format: `"{x} {y}"` or `"{x} {y} {areaError}"` — space-separated floats, validated in both storage backends
-- S3 key for a draft = `{Category}/{codename}.md`; `codename` = draft filename minus `DraftSuffix`
-- Section separator in drafts is a line that is exactly `...`; `---` is a kept horizontal rule
-- Telegram posting is append-only: only sections whose index is absent from the state file are posted
-- Semantic commit messages: `feat:`, `fix:`, `refactor:`
-- Structured logging via `log/slog`
diff --git a/config/config.go b/config/config.go
index 64d2e76..515047e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -10,67 +10,13 @@ import (
)
type Config struct {
- LogLevel slog.Level `json:"LogLevel" validate:"required"`
- MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"`
- DraftDir string `json:"DraftDir" validate:"required"`
- DraftSuffix string `json:"DraftSuffix" validate:"required"`
- Transcoders []TranscoderConfig `json:"Transcoders" validate:"required,min=1,dive"`
-}
-
-type TranscoderConfig struct {
- Type string `json:"Type" validate:"required,oneof=sayauz telegram"`
- Config any `json:"Config" validate:"required"`
-}
-
-func (tc *TranscoderConfig) UnmarshalJSON(data []byte) error {
- var tmp struct {
- Type string `json:"Type"`
- Config json.RawMessage `json:"Config"`
- }
-
- if err := json.Unmarshal(data, &tmp); err != nil {
- return err
- }
-
- tc.Type = tmp.Type
-
- switch tmp.Type {
- case "sayauz":
- var sayauzConfig SayauzConfig
- if err := json.Unmarshal(tmp.Config, &sayauzConfig); err != nil {
- return fmt.Errorf("unmarshal SayauzConfig: %w", err)
- }
- tc.Config = &sayauzConfig
- case "telegram":
- var telegramConfig TelegramConfig
- if err := json.Unmarshal(tmp.Config, &telegramConfig); err != nil {
- return fmt.Errorf("unmarshal TelegramConfig: %w", err)
- }
- tc.Config = &telegramConfig
- default:
- return fmt.Errorf("unsupported transcoder type: %s", tmp.Type)
- }
-
- return nil
-}
-
-type SayauzConfig struct {
- Category string `json:"Category" validate:"required,min=1"`
- Storage StorageConfig `json:"Storage" validate:"required"`
-}
-
-type TelegramConfig struct {
- BotToken string `json:"BotToken" validate:"required,min=1"`
- ChannelID int `json:"ChannelID" validate:"required,ne=0"`
- GroupID int `json:"GroupID"`
- ImageBaseURL string `json:"ImageBaseURL" validate:"required,min=1"`
- PreviewBaseURL string `json:"PreviewBaseURL"`
- PreviewExtension string `json:"PreviewExtension"`
- StateDir string `json:"StateDir" validate:"required,min=1"`
+ LogLevel slog.Level `json:"LogLevel" validate:"required"`
+ Storage StorageConfig `json:"Storage" validate:"required"`
+ MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"`
}
type StorageConfig struct {
- Type string `json:"Type" validate:"required,oneof=b2 s3"`
+ Type string `json:"Type" validate:"required,oneof=b2"`
Config any `json:"Config" validate:"required"`
}
@@ -93,12 +39,6 @@ func (sc *StorageConfig) UnmarshalJSON(data []byte) error {
return fmt.Errorf("unmarshal B2Config: %w", err)
}
sc.Config = &b2Config
- case "s3":
- var s3Config S3Config
- if err := json.Unmarshal(tmp.Config, &s3Config); err != nil {
- return fmt.Errorf("unmarshal S3Config: %w", err)
- }
- sc.Config = &s3Config
default:
return fmt.Errorf("unsupported storage type: %s", tmp.Type)
}
@@ -114,16 +54,6 @@ type B2Config struct {
ApplicationKey string `json:"ApplicationKey"`
}
-type S3Config struct {
- BucketName string `json:"BucketName" validate:"required,min=1"`
- Region string `json:"Region" validate:"required,min=1"`
- Prefix string `json:"Prefix"`
- Endpoint string `json:"Endpoint"`
- UsePathStyle bool `json:"UsePathStyle"`
- AccessKeyID string `json:"AccessKeyID"`
- SecretAccessKey string `json:"SecretAccessKey"`
-}
-
func LoadConfig(path string, config *Config) error {
fileBytes, err := os.ReadFile(path)
if err != nil {
diff --git a/config/config.json b/config/config.json
new file mode 100644
index 0000000..a48f0ff
--- /dev/null
+++ b/config/config.json
@@ -0,0 +1,14 @@
+{
+ "LogLevel": "debug",
+ "MaxConcurrentJobs": 8,
+ "Storage": {
+ "Type": "b2",
+ "Config": {
+ "BucketName": "sayana-pages",
+ "Region": "eu-central-003",
+ "Prefix": "",
+ "KeyID": "${B2_KEY_ID}",
+ "ApplicationKey": "${B2_APPLICATION_KEY}"
+ }
+ }
+} \ No newline at end of file
diff --git a/go.mod b/go.mod
index 276d23d..1b97d17 100644
--- a/go.mod
+++ b/go.mod
@@ -3,38 +3,15 @@ module github.com/SayaAndy/saya-today-article-metadata-add
go 1.24.5
require (
- github.com/Backblaze/blazer v0.7.2
- github.com/aws/aws-sdk-go-v2 v1.41.5
- github.com/aws/aws-sdk-go-v2/config v1.32.14
- github.com/aws/aws-sdk-go-v2/credentials v1.19.14
- github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0
- github.com/go-playground/validator/v10 v10.27.0
- github.com/yuin/goldmark v1.8.2
- gopkg.in/yaml.v3 v3.0.1
-)
-
-require (
- github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
- github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
- github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
- github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
- github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
- github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
- github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
- github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
- github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
- github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
- github.com/aws/smithy-go v1.24.2 // indirect
+ github.com/Backblaze/blazer v0.7.2 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
+ github.com/go-playground/validator/v10 v10.27.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
golang.org/x/crypto v0.33.0 // indirect
golang.org/x/net v0.34.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.22.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index 5be72ca..7984445 100644
--- a/go.sum
+++ b/go.sum
@@ -1,49 +1,7 @@
github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM=
github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10=
-github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
-github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
-github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
-github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
-github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
-github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
-github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
-github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
@@ -52,12 +10,6 @@ github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHO
github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
-github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
@@ -66,7 +18,6 @@ golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/draft/parser.go b/internal/draft/parser.go
deleted file mode 100644
index 2761c87..0000000
--- a/internal/draft/parser.go
+++ /dev/null
@@ -1,179 +0,0 @@
-package draft
-
-import (
- "regexp"
- "strings"
-
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
-)
-
-// layoutTokenRe matches gallery layout hints like "2x", "3x" that are
-// meaningful only to the sayauz renderer and must be dropped elsewhere.
-var layoutTokenRe = regexp.MustCompile(`^\d+x$`)
-
-type BlockKind int
-
-const (
- TextBlock BlockKind = iota
- GalleryBlock
-)
-
-// Block is one ordered piece of a section: either a run of markdown text or a
-// gallery. Exactly one of Text / Gallery is meaningful depending on Kind.
-type Block struct {
- Kind BlockKind
- Text string
- Gallery *Gallery
-}
-
-type Gallery struct {
- Timezone string
- KeyPrefix string
- Photos []Photo
-}
-
-type Photo struct {
- Key string // KeyPrefix + filename, e.g. "Sevsk/20241008-093833.jpg"
- Caption string
-}
-
-// Section is the content between two "..." separator lines, split into ordered
-// text and gallery blocks.
-type Section struct {
- Index int
- Blocks []Block
-}
-
-type Document struct {
- SourcePath string
- Codename string
- Metadata *frontmatter.Metadata
- RawContent []byte
- Body string
- Sections []Section
-}
-
-// ParseDraft parses a draft file: frontmatter metadata, the markdown body, and
-// the body split into sections (on "..." lines) of ordered text/gallery blocks.
-func ParseDraft(sourcePath, codename string, content []byte) (*Document, error) {
- metadata, body, err := frontmatter.ParseFrontmatter(content)
- if err != nil {
- return nil, err
- }
-
- doc := &Document{
- SourcePath: sourcePath,
- Codename: codename,
- Metadata: metadata,
- RawContent: content,
- Body: string(body),
- }
- doc.Sections = splitSections(doc.Body)
-
- return doc, nil
-}
-
-func splitSections(body string) []Section {
- lines := strings.Split(body, "\n")
-
- var sections []Section
- var chunk []string
-
- flush := func() {
- blocks := parseBlocks(chunk)
- chunk = nil
- if len(blocks) == 0 {
- return
- }
- sections = append(sections, Section{Index: len(sections), Blocks: blocks})
- }
-
- for _, line := range lines {
- if strings.TrimSpace(line) == "..." {
- flush()
- continue
- }
- chunk = append(chunk, line)
- }
- flush()
-
- return sections
-}
-
-func parseBlocks(lines []string) []Block {
- var blocks []Block
- var text []string
-
- flushText := func() {
- joined := strings.TrimSpace(strings.Join(text, "\n"))
- text = nil
- if joined != "" {
- blocks = append(blocks, Block{Kind: TextBlock, Text: joined})
- }
- }
-
- for i := 0; i < len(lines); i++ {
- trimmed := strings.TrimSpace(lines[i])
- if strings.HasPrefix(trimmed, "{Gallery:") {
- flushText()
- gallery, next := parseGallery(lines, i)
- if gallery != nil {
- blocks = append(blocks, Block{Kind: GalleryBlock, Gallery: gallery})
- }
- i = next
- continue
- }
- text = append(text, lines[i])
- }
- flushText()
-
- return blocks
-}
-
-// parseGallery reads a {Gallery:...} block starting at lines[start] and returns
-// the parsed gallery plus the index of the closing {/Gallery} line (or the last
-// consumed line if unterminated).
-func parseGallery(lines []string, start int) (*Gallery, int) {
- header := strings.TrimSpace(lines[start])
- header = strings.TrimPrefix(header, "{Gallery:")
- header = strings.TrimSuffix(header, "}")
- tz, keyPrefix, _ := strings.Cut(header, ":")
-
- gallery := &Gallery{Timezone: tz, KeyPrefix: keyPrefix}
-
- i := start + 1
- for ; i < len(lines); i++ {
- trimmed := strings.TrimSpace(lines[i])
- if trimmed == "{/Gallery}" {
- break
- }
- if trimmed == "" {
- continue
- }
- gallery.Photos = append(gallery.Photos, parsePhoto(keyPrefix, trimmed))
- }
-
- return gallery, i
-}
-
-func parsePhoto(keyPrefix, line string) Photo {
- parts := strings.Split(line, "|")
- for i := range parts {
- parts[i] = strings.TrimSpace(parts[i])
- }
-
- filename := parts[0]
-
- var captionParts []string
- for _, p := range parts[1:] {
- if p == "" || layoutTokenRe.MatchString(p) {
- continue
- }
- captionParts = append(captionParts, p)
- }
-
- return Photo{
- Key: keyPrefix + filename,
- Caption: strings.Join(captionParts, " "),
- }
-}
diff --git a/internal/draft/parser_test.go b/internal/draft/parser_test.go
deleted file mode 100644
index c6d454d..0000000
--- a/internal/draft/parser_test.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package draft
-
-import (
- "os"
- "strings"
- "testing"
-)
-
-func TestParseDraftSevsk(t *testing.T) {
- raw, err := os.ReadFile("../../.draft/sevsk.md.draft")
- if err != nil {
- t.Fatal(err)
- }
-
- doc, err := ParseDraft("../../.draft/sevsk.md.draft", "sevsk", raw)
- if err != nil {
- t.Fatal(err)
- }
- if doc.Metadata == nil || doc.Metadata.Title == "" {
- t.Fatal("expected frontmatter metadata")
- }
-
- var galleries int
- for _, section := range doc.Sections {
- for _, block := range section.Blocks {
- if block.Kind != GalleryBlock {
- continue
- }
- galleries++
- for _, p := range block.Gallery.Photos {
- if !strings.HasPrefix(p.Key, "Sevsk/20241008-") {
- t.Errorf("unexpected photo key %q", p.Key)
- }
- if strings.Contains(p.Caption, "2x") {
- t.Errorf("layout token leaked into caption %q", p.Caption)
- }
- }
- }
- }
- if galleries == 0 {
- t.Fatal("expected at least one gallery")
- }
-
- // First gallery: 4 photos, captions on photos 1 and 3, none on 2; photo 4 had "2x |" => empty caption.
- first := firstGallery(doc)
- if first == nil || len(first.Photos) != 4 {
- t.Fatalf("first gallery: want 4 photos, got %v", first)
- }
- if first.Timezone != "Europe/Moscow" {
- t.Errorf("first gallery tz = %q", first.Timezone)
- }
- if first.Photos[0].Caption != "Никольская церковь. Действующая" {
- t.Errorf("photo[0] caption = %q", first.Photos[0].Caption)
- }
- if first.Photos[1].Caption != "" {
- t.Errorf("photo[1] caption = %q, want empty", first.Photos[1].Caption)
- }
- if first.Photos[3].Caption != "" {
- t.Errorf("photo[3] caption = %q, want empty (only 2x token)", first.Photos[3].Caption)
- }
- if first.Photos[3].Key != "Sevsk/20241008-094036.jpg" {
- t.Errorf("photo[3] key = %q", first.Photos[3].Key)
- }
-}
-
-func firstGallery(doc *Document) *Gallery {
- for _, section := range doc.Sections {
- for _, block := range section.Blocks {
- if block.Kind == GalleryBlock {
- return block.Gallery
- }
- }
- }
- return nil
-}
-
-func TestSplitSectionsSeparator(t *testing.T) {
- body := "intro\n...\nmiddle\n...\nend"
- sections := splitSections(body)
- if len(sections) != 3 {
- t.Fatalf("want 3 sections, got %d", len(sections))
- }
-}
diff --git a/internal/frontmatter/parser.go b/internal/frontmatter/parser.go
index 299d4ee..20512d9 100644
--- a/internal/frontmatter/parser.go
+++ b/internal/frontmatter/parser.go
@@ -16,8 +16,7 @@ type Metadata struct {
Thumbnail string `yaml:"thumbnail"`
Tags []string `yaml:"tags"`
Geolocation string `yaml:"geolocation"`
- Medley string `yaml:"medley"`
- MedleyPart int `yaml:"medleyPart"`
+ Timezone string `yaml:"timezone"`
}
func ParseFrontmatter(content []byte) (metadata *Metadata, markdown []byte, err error) {
diff --git a/internal/storage/b2.go b/internal/storage/b2.go
index 6697023..0d92707 100644
--- a/internal/storage/b2.go
+++ b/internal/storage/b2.go
@@ -3,6 +3,7 @@ package storage
import (
"context"
"fmt"
+ "io"
"strconv"
"strings"
"time"
@@ -39,22 +40,63 @@ func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
}
-func (sc *B2StorageClient) GetMetadata(key string) (map[string]string, error) {
- obj := sc.bucket.Object(sc.prefix + key)
+func (sc *B2StorageClient) Scan() ([]string, error) {
+ filePaths := []string{}
+
+ iter := sc.bucket.List(context.Background(), b2.ListPrefix(sc.prefix))
+
+ for iter.Next() {
+ obj := iter.Object()
+ if obj == nil {
+ return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ }
+
+ attrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("get attributes for object: %w", err)
+ }
+
+ if attrs.Status != b2.Uploaded {
+ continue
+ }
+
+ if !strings.HasSuffix(obj.Name(), ".md") {
+ continue
+ }
+
+ filePaths = append(filePaths, strings.TrimPrefix(obj.Name(), sc.prefix))
+ }
+
+ if err := iter.Err(); err != nil {
+ return nil, fmt.Errorf("iterate over B2 objects: %w", err)
+ }
+
+ return filePaths, nil
+}
+
+func (sc *B2StorageClient) GetReader(path string) (io.ReadCloser, int64, error) {
+ obj := sc.bucket.Object(sc.prefix + path)
if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ return nil, 0, fmt.Errorf("failed to reference object in B2 bucket")
}
attrs, err := obj.Attrs(context.Background())
if err != nil {
- if b2.IsNotExist(err) {
- return map[string]string{}, nil
- }
- return nil, fmt.Errorf("get attributes of B2 object %q: %w", sc.prefix+key, err)
+ return nil, 0, fmt.Errorf("error getting attributes of an object: %w", err)
}
- return attrs.Info, nil
+
+ return obj.NewReader(context.Background()), attrs.Size, nil
}
-func (sc *B2StorageClient) Put(key string, content []byte, metadata *frontmatter.Metadata) error {
+func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error {
+ obj := sc.bucket.Object(sc.prefix + path)
+ if obj == nil {
+ return fmt.Errorf("failed to reference object in B2 bucket")
+ }
+ oldAttrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return fmt.Errorf("error getting attributes of an object: %w", err)
+ }
+
geolocationParts := strings.Split(metadata.Geolocation, " ")
if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 {
return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string")
@@ -73,30 +115,27 @@ func (sc *B2StorageClient) Put(key string, content []byte, metadata *frontmatter
}
}
- medley := ""
- if metadata.Medley != "" {
- medley = fmt.Sprintf("%s %d", metadata.Medley, metadata.MedleyPart)
- }
-
attrs := &b2.Attrs{
ContentType: "text/markdown; charset=utf-8",
Info: map[string]string{
- "title": metadata.Title,
- "short-description": metadata.ShortDescription,
- "action-date": metadata.ActionDate,
- "published-time": metadata.PublishedTime.Format(time.RFC3339),
- "thumbnail": metadata.Thumbnail,
- "tags": strings.Join(metadata.Tags, ","),
- "geolocation": metadata.Geolocation,
- "medley": medley,
+ "title": metadata.Title,
+ "short-description": metadata.ShortDescription,
+ "action-date": metadata.ActionDate,
+ "published-time": metadata.PublishedTime.Format(time.RFC3339),
+ "thumbnail": metadata.Thumbnail,
+ "tags": strings.Join(metadata.Tags, ","),
+ "geolocation": metadata.Geolocation,
+ "timezone": metadata.Timezone,
+ "metadata-last-update-sha1": oldAttrs.SHA1,
}}
- prod := sc.bucket.Object(sc.prefix + key)
- if prod == nil {
- return fmt.Errorf("failed to reference prod object in B2 bucket")
+ reader := obj.NewReader(context.Background())
+ content := make([]byte, oldAttrs.Size)
+ if _, err = reader.Read(content); err != nil {
+ return fmt.Errorf("failed to read an object back for writing (required for attribute setting): %w", err)
}
- writer := prod.NewWriter(context.Background(), b2.WithAttrsOption(attrs))
+ writer := obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs))
defer writer.Close()
if _, err := writer.Write(content); err != nil {
return fmt.Errorf("failed to write an object back after attribute settings: %w", err)
@@ -105,6 +144,21 @@ func (sc *B2StorageClient) Put(key string, content []byte, metadata *frontmatter
return nil
}
-func (sc *B2StorageClient) BuildIndex() error {
- return fmt.Errorf("BuildIndex not implemented for B2 storage")
+func (sc *B2StorageClient) FileHasChanged(path string) bool {
+ obj := sc.bucket.Object(sc.prefix + path)
+ if obj == nil {
+ return true
+ }
+
+ attrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return true
+ }
+
+ lastUpdateSha1, ok := attrs.Info["metadata-last-update-sha1"]
+ if !ok {
+ return true
+ }
+
+ return attrs.SHA1 == lastUpdateSha1
}
diff --git a/internal/storage/index.go b/internal/storage/index.go
deleted file mode 100644
index bfeafbe..0000000
--- a/internal/storage/index.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package storage
-
-import (
- "encoding/json"
- "fmt"
- "time"
-)
-
-const IndexFileName = "index.json"
-const MedleysIndexFileName = "medleys.json"
-
-const IndexSchemaVersion = 2
-
-type IndexEntry struct {
- Link string `json:"link"`
- ModifiedTime time.Time `json:"modifiedTime"`
- Title string `json:"title"`
- ShortDescription string `json:"shortDescription"`
- ActionDate string `json:"actionDate"`
- PublishedTime time.Time `json:"publishedTime"`
- Thumbnail string `json:"thumbnail"`
- Tags []string `json:"tags"`
- Geolocation string `json:"geolocation"`
- Medley string `json:"medley,omitempty"`
- MedleyPart int `json:"medleyPart,omitempty"`
-}
-
-type IndexV2Category struct {
- GeneratedAt time.Time `json:"generatedAt"`
- Pages map[string]IndexEntry `json:"pages"`
-}
-
-type IndexV1Category struct {
- GeneratedAt time.Time `json:"generatedAt"`
- Pages []IndexEntry `json:"pages"`
-}
-
-type Index struct {
- SchemaVersion int `json:"schemaVersion"`
- GeneratedAt time.Time `json:"generatedAt"`
- Categories any `json:"categories"`
-}
-
-func (idx *Index) UnmarshalJSON(data []byte) error {
- var tmp struct {
- SchemaVersion int `json:"schemaVersion"`
- GeneratedAt time.Time `json:"generatedAt"`
- Categories json.RawMessage `json:"categories"`
- }
-
- if err := json.Unmarshal(data, &tmp); err != nil {
- return err
- }
-
- idx.SchemaVersion = tmp.SchemaVersion
- idx.GeneratedAt = tmp.GeneratedAt
-
- switch tmp.SchemaVersion {
- case 1:
- var categories map[string]*IndexV1Category
- if err := json.Unmarshal(tmp.Categories, &categories); err != nil {
- return fmt.Errorf("unmarshal map[string]*IndexV1Category: %w", err)
- }
- idx.Categories = &categories
- case 2:
- var categories map[string]*IndexV2Category
- if err := json.Unmarshal(tmp.Categories, &categories); err != nil {
- return fmt.Errorf("unmarshal map[string]*IndexV2Category: %w", err)
- }
- idx.Categories = &categories
- default:
- return fmt.Errorf("unsupported index version: %d", tmp.SchemaVersion)
- }
-
- return nil
-}
-
-type IndexV1 struct {
- SchemaVersion int `json:"schemaVersion"`
- GeneratedAt time.Time `json:"generatedAt"`
- Categories map[string]*IndexV1Category `json:"categories"`
-}
-
-type MedleyEntry struct {
- Codename string `json:"codename"`
- Content []string `json:"content"`
-}
-
-type MedleyPageEntry struct {
- Codename string `json:"codename"`
- Position int `json:"position"`
-}
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
deleted file mode 100644
index 5f3acc7..0000000
--- a/internal/storage/s3.go
+++ /dev/null
@@ -1,389 +0,0 @@
-package storage
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "log/slog"
- "maps"
- "net/url"
- "slices"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "github.com/aws/aws-sdk-go-v2/aws"
- awsconfig "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
- s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
-
- "github.com/SayaAndy/saya-today-article-metadata-add/config"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
-)
-
-const s3IndexConcurrency = 32
-
-var _ StorageClient = &S3StorageClient{}
-
-type S3StorageClient struct {
- prefix string
- bucket string
- client *s3.Client
-}
-
-func NewS3StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
- if cfg.Type != "s3" {
- return nil, fmt.Errorf("invalid storage type for S3StorageClient")
- }
- s3cfg := cfg.Config.(*config.S3Config)
-
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(s3cfg.Region),
- }
- if s3cfg.AccessKeyID != "" && s3cfg.SecretAccessKey != "" {
- opts = append(opts, awsconfig.WithCredentialsProvider(
- credentials.NewStaticCredentialsProvider(s3cfg.AccessKeyID, s3cfg.SecretAccessKey, ""),
- ))
- }
-
- awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
- if err != nil {
- return nil, fmt.Errorf("load AWS config: %w", err)
- }
-
- var s3Opts []func(*s3.Options)
- if s3cfg.Endpoint != "" {
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.BaseEndpoint = aws.String(s3cfg.Endpoint)
- })
- }
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.UsePathStyle = s3cfg.UsePathStyle
- })
-
- client := s3.NewFromConfig(awsCfg, s3Opts...)
-
- return &S3StorageClient{
- client: client,
- bucket: s3cfg.BucketName,
- prefix: s3cfg.Prefix,
- }, nil
-}
-
-func (sc *S3StorageClient) GetMetadata(key string) (map[string]string, error) {
- head, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(sc.prefix + key),
- })
- if err != nil {
- var nsk *s3types.NoSuchKey
- var nf *s3types.NotFound
- if errors.As(err, &nsk) || errors.As(err, &nf) {
- return map[string]string{}, nil
- }
- return nil, fmt.Errorf("head S3 object %q: %w", sc.prefix+key, err)
- }
- return head.Metadata, nil
-}
-
-func (sc *S3StorageClient) Put(key string, content []byte, metadata *frontmatter.Metadata) error {
- geolocationParts := strings.Split(metadata.Geolocation, " ")
- if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 {
- return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string")
- }
- if len(geolocationParts) >= 2 {
- if _, err := strconv.ParseFloat(geolocationParts[0], 64); err != nil {
- return fmt.Errorf("invalid geolocation parameter, expected float for X: %w", err)
- }
- if _, err := strconv.ParseFloat(geolocationParts[1], 64); err != nil {
- return fmt.Errorf("invalid geolocation parameter, expected float for Y: %w", err)
- }
- }
- if len(geolocationParts) == 3 {
- if _, err := strconv.ParseFloat(geolocationParts[2], 64); err != nil {
- return fmt.Errorf("invalid geolocation parameter, expected float for area error: %w", err)
- }
- }
-
- s3Metadata := map[string]string{
- "title": url.QueryEscape(metadata.Title),
- "short-description": url.QueryEscape(metadata.ShortDescription),
- "action-date": metadata.ActionDate,
- "published-time": metadata.PublishedTime.Format(time.RFC3339),
- "thumbnail": url.QueryEscape(metadata.Thumbnail),
- "tags": strings.Join(metadata.Tags, ","),
- "geolocation": metadata.Geolocation,
- }
-
- targetKey := sc.prefix + key
- _, err := sc.client.PutObject(context.Background(), &s3.PutObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(targetKey),
- Body: bytes.NewReader(content),
- ContentType: aws.String("text/markdown; charset=utf-8"),
- Metadata: s3Metadata,
- })
- if err != nil {
- return fmt.Errorf("put S3 object %q: %w", targetKey, err)
- }
-
- return nil
-}
-
-func (sc *S3StorageClient) BuildIndex() error {
- type candidate struct {
- key string
- codename string
- lastModified time.Time
- }
- var candidates []candidate
-
- paginator := s3.NewListObjectsV2Paginator(sc.client, &s3.ListObjectsV2Input{
- Bucket: aws.String(sc.bucket),
- Prefix: aws.String(sc.prefix),
- })
- for paginator.HasMorePages() {
- page, err := paginator.NextPage(context.Background())
- if err != nil {
- return fmt.Errorf("list S3 objects: %w", err)
- }
- for _, obj := range page.Contents {
- key := aws.ToString(obj.Key)
- if key == IndexFileName {
- continue
- }
- if !strings.HasSuffix(key, ".md") {
- continue
- }
- codename := key[strings.LastIndex(key, "/")+1 : strings.LastIndex(key, ".")]
- candidates = append(candidates, candidate{key, codename, aws.ToTime(obj.LastModified)})
- }
- }
-
- medleys, err := sc.scanMedleys()
- if err != nil {
- slog.Warn("skipped reading medleys due to an error", slog.String("error", err.Error()))
- }
- pageToMedleyMap := make(map[string]MedleyPageEntry)
- for _, medley := range medleys {
- for i, page := range medley.Content {
- pageToMedleyMap[page] = MedleyPageEntry{medley.Codename, i}
- }
- }
-
- type result struct {
- catKey string
- codename string
- entry IndexEntry
- }
- results := make([]*result, len(candidates))
- sem := make(chan struct{}, s3IndexConcurrency)
- var wg sync.WaitGroup
- var firstErr error
- var errMu sync.Mutex
-
- for i, cand := range candidates {
- sem <- struct{}{}
- wg.Add(1)
- go func() {
- defer wg.Done()
- defer func() { <-sem }()
-
- head, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(cand.key),
- })
- if err != nil {
- errMu.Lock()
- if firstErr == nil {
- firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err)
- }
- errMu.Unlock()
- return
- }
-
- if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") {
- return
- }
- meta := head.Metadata
- if meta["title"] == "" {
- return
- }
-
- publishedTime, err := time.Parse(time.RFC3339, meta["published-time"])
- if err != nil {
- slog.Warn("skip entry with bad published-time", slog.String("key", cand.key), slog.String("error", err.Error()))
- return
- }
-
- catKey := cand.key[:strings.Index(cand.key, "/")]
-
- tags := strings.Split(meta["tags"], ",")
- slices.Sort(tags)
-
- title, _ := url.QueryUnescape(meta["title"])
- shortDescription, _ := url.QueryUnescape(meta["short-description"])
- thumbnail, _ := url.QueryUnescape(meta["thumbnail"])
-
- medleyName, medleyPart := "", 0
- if medley, ok := pageToMedleyMap[cand.codename]; ok {
- medleyName, medleyPart = medley.Codename, medley.Position
- }
-
- results[i] = &result{
- catKey: catKey,
- codename: cand.codename,
- entry: IndexEntry{
- Link: cand.key,
- ModifiedTime: cand.lastModified,
- Title: title,
- ShortDescription: shortDescription,
- ActionDate: meta["action-date"],
- PublishedTime: publishedTime,
- Thumbnail: thumbnail,
- Tags: tags,
- Geolocation: meta["geolocation"],
- Medley: medleyName,
- MedleyPart: medleyPart,
- },
- }
- }()
- }
- wg.Wait()
-
- if firstErr != nil {
- return firstErr
- }
-
- now := time.Now().UTC()
- fresh := make(map[string]*IndexV2Category)
- for _, r := range results {
- if r == nil {
- continue
- }
- if _, ok := fresh[r.catKey]; !ok {
- fresh[r.catKey] = &IndexV2Category{
- Pages: make(map[string]IndexEntry),
- }
- }
- fresh[r.catKey].Pages[r.codename] = r.entry
- fresh[r.catKey].GeneratedAt = now
- }
-
- merged, err := sc.loadAndMergeIndex(fresh)
- if err != nil {
- return err
- }
-
- idx := Index{
- SchemaVersion: IndexSchemaVersion,
- GeneratedAt: now,
- Categories: merged,
- }
-
- body, err := json.Marshal(idx)
- if err != nil {
- return fmt.Errorf("marshal index: %w", err)
- }
-
- _, err = sc.client.PutObject(context.Background(), &s3.PutObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(IndexFileName),
- Body: bytes.NewReader(body),
- ContentType: aws.String("application/json; charset=utf-8"),
- })
- if err != nil {
- return fmt.Errorf("put index %q: %w", IndexFileName, err)
- }
-
- totalEntries := 0
- for _, c := range merged {
- totalEntries += len(c.Pages)
- }
- slog.Info("wrote index",
- slog.String("key", IndexFileName),
- slog.Int("categories", len(merged)),
- slog.Int("entries", totalEntries))
- return nil
-}
-
-func (sc *S3StorageClient) scanMedleys() ([]MedleyEntry, error) {
- out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(MedleysIndexFileName),
- })
- if err != nil {
- return nil, fmt.Errorf("get %s: %w", MedleysIndexFileName, err)
- }
- defer out.Body.Close()
-
- raw, err := io.ReadAll(out.Body)
- if err != nil {
- return nil, fmt.Errorf("read %s: %w", MedleysIndexFileName, err)
- }
-
- var entries []MedleyEntry
- if err := json.Unmarshal(raw, &entries); err != nil {
- return nil, fmt.Errorf("unmarshal %s: %w", MedleysIndexFileName, err)
- }
-
- return entries, nil
-}
-
-func (sc *S3StorageClient) loadAndMergeIndex(fresh map[string]*IndexV2Category) (map[string]*IndexV2Category, error) {
- merged := make(map[string]*IndexV2Category)
-
- out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(IndexFileName),
- })
- if err == nil {
- defer out.Body.Close()
- raw, readErr := io.ReadAll(out.Body)
- if readErr != nil {
- return nil, fmt.Errorf("read existing index: %w", readErr)
- }
-
- var legacyIdx Index
- if err := json.Unmarshal(raw, &legacyIdx); err != nil {
- return nil, fmt.Errorf("unmarshal existing index: %w", err)
- }
-
- switch legacyIdx.SchemaVersion {
- case 1:
- for k, v := range *legacyIdx.Categories.(*map[string]*IndexV1Category) {
- pages := make(map[string]IndexEntry, len(v.Pages))
- for _, page := range v.Pages {
- pages[page.Link[strings.LastIndex(page.Link, "/")+1:strings.LastIndex(page.Link, ".")]] = page
- }
- merged[k] = &IndexV2Category{
- GeneratedAt: v.GeneratedAt,
- Pages: pages,
- }
- }
- case 2:
- maps.Copy(merged, *legacyIdx.Categories.(*map[string]*IndexV2Category))
- default:
- slog.Warn("unknown index schema, discarding", slog.Int("schema_version", legacyIdx.SchemaVersion))
- }
- } else {
- var nsk *s3types.NoSuchKey
- if !errors.As(err, &nsk) {
- return nil, fmt.Errorf("get existing index: %w", err)
- }
- }
-
- for k := range merged {
- if strings.HasPrefix(k, sc.prefix) {
- delete(merged, k)
- }
- }
- maps.Copy(merged, fresh)
-
- return merged, nil
-}
diff --git a/internal/storage/storage_interface.go b/internal/storage/storage_interface.go
index 92a37e2..91eeedb 100644
--- a/internal/storage/storage_interface.go
+++ b/internal/storage/storage_interface.go
@@ -1,17 +1,19 @@
package storage
import (
+ "io"
+
"github.com/SayaAndy/saya-today-article-metadata-add/config"
"github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
)
type StorageClient interface {
- Put(key string, content []byte, metadata *frontmatter.Metadata) error
- GetMetadata(key string) (metadata map[string]string, err error)
- BuildIndex() error
+ Scan() (paths []string, err error)
+ GetReader(path string) (reader io.ReadCloser, sz int64, err error)
+ WriteMetadata(path string, metadata *frontmatter.Metadata) error
+ FileHasChanged(path string) bool
}
-var NewStorageClientMap = map[string]func(*config.StorageConfig) (StorageClient, error){
+var NewStorageClientMap = map[string]func(cfg *config.StorageConfig) (StorageClient, error){
"b2": NewB2StorageClient,
- "s3": NewS3StorageClient,
}
diff --git a/internal/transcoder/format_test.go b/internal/transcoder/format_test.go
deleted file mode 100644
index 9b25dc6..0000000
--- a/internal/transcoder/format_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package transcoder
-
-import (
- "os"
- "testing"
- "time"
-)
-
-func TestFormatContentMatchesCommitted(t *testing.T) {
- draftRaw, err := os.ReadFile("../../.draft/sevsk.md.draft")
- if err != nil {
- t.Fatal(err)
- }
- expected, err := os.ReadFile("../../.draft/sevsk.md")
- if err != nil {
- t.Fatal(err)
- }
-
- published, err := time.Parse(time.RFC3339, "2025-08-26T11:21:53+07:00")
- if err != nil {
- t.Fatal(err)
- }
-
- got := formatContent(draftRaw, published, true)
- if string(got) != string(expected) {
- t.Errorf("formatted output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, expected)
- }
-}
diff --git a/internal/transcoder/markdown.go b/internal/transcoder/markdown.go
deleted file mode 100644
index 975c3bc..0000000
--- a/internal/transcoder/markdown.go
+++ /dev/null
@@ -1,164 +0,0 @@
-package transcoder
-
-import (
- "fmt"
- "strings"
-
- "github.com/yuin/goldmark"
- "github.com/yuin/goldmark/ast"
- "github.com/yuin/goldmark/extension"
- xast "github.com/yuin/goldmark/extension/ast"
- "github.com/yuin/goldmark/text"
-)
-
-// telegramMD parses CommonMark once and is reused across calls.
-var telegramMD = goldmark.New(goldmark.WithExtensions(extension.Strikethrough, extension.Linkify))
-
-// markdownToTelegramHTML converts CommonMark source into the limited HTML
-// subset Telegram accepts (parse_mode=HTML). Telegram supports only
-// b/i/u/s/a/code/pre/blockquote — block constructs with no Telegram tag
-// (paragraphs, headings, lists) are flattened to text + newlines. Anything
-// outside the subset is dropped rather than emitted, so a message is never
-// rejected for an unsupported tag.
-func markdownToTelegramHTML(src string) string {
- source := []byte(src)
- doc := telegramMD.Parser().Parse(text.NewReader(source))
-
- var b strings.Builder
- renderNodes(&b, doc, source)
-
- // Collapse the runs of blank lines block rendering can leave behind.
- out := strings.TrimSpace(b.String())
- for strings.Contains(out, "\n\n\n") {
- out = strings.ReplaceAll(out, "\n\n\n", "\n\n")
- }
- return out
-}
-
-func renderNodes(b *strings.Builder, parent ast.Node, source []byte) {
- for n := parent.FirstChild(); n != nil; n = n.NextSibling() {
- renderNode(b, n, source)
- }
-}
-
-func renderNode(b *strings.Builder, n ast.Node, source []byte) {
- switch node := n.(type) {
- case *ast.Document:
- renderNodes(b, node, source)
-
- case *ast.Paragraph, *ast.TextBlock:
- renderNodes(b, node, source)
- b.WriteString("\n\n")
-
- case *ast.Heading:
- // Telegram has no headings; render the line in bold.
- b.WriteString("<b>")
- renderNodes(b, node, source)
- b.WriteString("</b>\n\n")
-
- case *ast.Blockquote:
- b.WriteString("<blockquote>")
- renderNodes(b, node, source)
- trimTrailingNewlines(b)
- b.WriteString("</blockquote>\n\n")
-
- case *ast.List:
- renderList(b, node, source)
- b.WriteString("\n")
-
- case *ast.FencedCodeBlock, *ast.CodeBlock:
- b.WriteString("<pre>")
- writeRawLines(b, n, source)
- b.WriteString("</pre>\n\n")
-
- case *ast.ThematicBreak:
- // horizontal rule — nothing meaningful in a Telegram message
-
- // --- inline ---
- case *ast.Text:
- b.WriteString(escapeHTML(string(node.Segment.Value(source))))
- if node.HardLineBreak() || node.SoftLineBreak() {
- b.WriteByte('\n')
- }
- case *ast.String:
- b.WriteString(escapeHTML(string(node.Value)))
-
- case *ast.Emphasis:
- tag := "i"
- if node.Level == 2 {
- tag = "b"
- }
- fmt.Fprintf(b, "<%s>", tag)
- renderNodes(b, node, source)
- fmt.Fprintf(b, "</%s>", tag)
-
- case *xast.Strikethrough:
- b.WriteString("<s>")
- renderNodes(b, node, source)
- b.WriteString("</s>")
-
- case *ast.CodeSpan:
- b.WriteString("<code>")
- renderNodes(b, node, source)
- b.WriteString("</code>")
-
- case *ast.Link:
- fmt.Fprintf(b, `<a href="%s">`, escapeHTML(string(node.Destination)))
- renderNodes(b, node, source)
- b.WriteString("</a>")
-
- case *ast.AutoLink:
- url := string(node.URL(source))
- fmt.Fprintf(b, `<a href="%s">%s</a>`, escapeHTML(url), escapeHTML(url))
-
- case *ast.Image:
- // Images can't render inline in text; keep the alt text only.
- renderNodes(b, node, source)
-
- case *ast.RawHTML, *ast.HTMLBlock:
- // Drop raw HTML — it is almost certainly not in Telegram's tag subset.
-
- default:
- // Unknown node: recurse so inline text inside it is not lost.
- renderNodes(b, n, source)
- }
-}
-
-func renderList(b *strings.Builder, list *ast.List, source []byte) {
- i := list.Start
- for item := list.FirstChild(); item != nil; item = item.NextSibling() {
- if list.IsOrdered() {
- fmt.Fprintf(b, "%d. ", i)
- i++
- } else {
- b.WriteString("• ")
- }
- renderNodes(b, item, source)
- trimTrailingNewlines(b)
- b.WriteByte('\n')
- }
-}
-
-func writeRawLines(b *strings.Builder, n ast.Node, source []byte) {
- lines := n.Lines()
- for i := 0; i < lines.Len(); i++ {
- seg := lines.At(i)
- b.WriteString(escapeHTML(string(seg.Value(source))))
- }
-}
-
-func trimTrailingNewlines(b *strings.Builder) {
- s := strings.TrimRight(b.String(), "\n")
- b.Reset()
- b.WriteString(s)
-}
-
-// escapeHTML escapes the three characters Telegram's HTML parser treats as
-// markup. Quotes are escaped too so the value is safe inside an href="...".
-func escapeHTML(s string) string {
- s = strings.ReplaceAll(s, "&", "&amp;")
- s = strings.ReplaceAll(s, "<", "&lt;")
- s = strings.ReplaceAll(s, ">", "&gt;")
- s = strings.ReplaceAll(s, `"`, "&quot;")
- return s
-}
diff --git a/internal/transcoder/sayauz.go b/internal/transcoder/sayauz.go
deleted file mode 100644
index ee888e5..0000000
--- a/internal/transcoder/sayauz.go
+++ /dev/null
@@ -1,138 +0,0 @@
-package transcoder
-
-import (
- "fmt"
- "log/slog"
- "os"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/SayaAndy/saya-today-article-metadata-add/config"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage"
-)
-
-type SayauzTranscoder struct {
- category string
- storage storage.StorageClient
-}
-
-func NewSayauzTranscoder(cfg any) (Transcoder, error) {
- sayauzCfg, ok := cfg.(*config.SayauzConfig)
- if !ok {
- return nil, fmt.Errorf("invalid config for sayauz transcoder")
- }
-
- newClient, ok := storage.NewStorageClientMap[sayauzCfg.Storage.Type]
- if !ok {
- return nil, fmt.Errorf("unsupported storage type %q for sayauz transcoder", sayauzCfg.Storage.Type)
- }
- storageClient, err := newClient(&sayauzCfg.Storage)
- if err != nil {
- return nil, fmt.Errorf("init storage client: %w", err)
- }
-
- return &SayauzTranscoder{
- category: sayauzCfg.Category,
- storage: storageClient,
- }, nil
-}
-
-func (t *SayauzTranscoder) Name() string { return "sayauz" }
-
-func (t *SayauzTranscoder) Transcode(doc *draft.Document) error {
- if doc.Metadata == nil {
- return fmt.Errorf("draft %q has no frontmatter metadata", doc.SourcePath)
- }
-
- prodKey := t.category + "/" + doc.Codename + ".md"
-
- // publishedTime is the first-publish time: reuse an existing value (from the
- // draft frontmatter, else from the already-published object), otherwise stamp now.
- inject := doc.Metadata.PublishedTime.IsZero()
- if inject {
- published := time.Time{}
- if meta, err := t.storage.GetMetadata(prodKey); err == nil {
- if v := meta["published-time"]; v != "" {
- if parsed, err := time.Parse(time.RFC3339, v); err == nil {
- published = parsed
- }
- }
- } else {
- slog.Warn("could not read existing metadata for publishedTime, stamping now",
- slog.String("key", prodKey), slog.String("error", err.Error()))
- }
- if published.IsZero() {
- published = time.Now()
- }
- doc.Metadata.PublishedTime = published
- }
-
- formatted := formatContent(doc.RawContent, doc.Metadata.PublishedTime, inject)
-
- localPath := filepath.Join(filepath.Dir(doc.SourcePath), doc.Codename+".md")
- if err := os.WriteFile(localPath, formatted, 0o644); err != nil {
- return fmt.Errorf("write formatted file %q: %w", localPath, err)
- }
-
- if err := t.storage.Put(prodKey, formatted, doc.Metadata); err != nil {
- return fmt.Errorf("upload %q: %w", prodKey, err)
- }
-
- slog.Info("sayauz published page",
- slog.String("key", prodKey),
- slog.String("local", localPath))
- return nil
-}
-
-func (t *SayauzTranscoder) Finalize() error {
- return t.storage.BuildIndex()
-}
-
-// formatContent renders a draft into its published form: standalone "..."
-// separator lines become blank lines (trailing ones are dropped), and when
-// inject is true a "publishedTime:" line is added after "actionDate:" in the
-// frontmatter. Frontmatter, galleries and "---" rules are otherwise untouched.
-func formatContent(raw []byte, published time.Time, inject bool) []byte {
- lines := strings.Split(string(raw), "\n")
- out := make([]string, 0, len(lines)+1)
-
- inFrontmatter := false
- frontmatterDone := false
-
- for i, line := range lines {
- trimmed := strings.TrimSpace(line)
-
- if !frontmatterDone {
- if i == 0 && trimmed == "---" {
- inFrontmatter = true
- out = append(out, line)
- continue
- }
- if inFrontmatter {
- out = append(out, line)
- switch {
- case trimmed == "---":
- inFrontmatter = false
- frontmatterDone = true
- case inject && strings.HasPrefix(trimmed, "actionDate:"):
- out = append(out, "publishedTime: "+published.Format(time.RFC3339))
- }
- continue
- }
- }
-
- if trimmed == "..." {
- out = append(out, "")
- continue
- }
- out = append(out, line)
- }
-
- for len(out) > 0 && out[len(out)-1] == "" {
- out = out[:len(out)-1]
- }
-
- return []byte(strings.Join(out, "\n"))
-}
diff --git a/internal/transcoder/telegram.go b/internal/transcoder/telegram.go
deleted file mode 100644
index b521fb0..0000000
--- a/internal/transcoder/telegram.go
+++ /dev/null
@@ -1,584 +0,0 @@
-package transcoder
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log/slog"
- "mime/multipart"
- "net/http"
- "os"
- "path/filepath"
- "slices"
- "strconv"
- "strings"
- "time"
-
- "github.com/SayaAndy/saya-today-article-metadata-add/config"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft"
-)
-
-const (
- telegramMediaGroupLimit = 10
- // telegramCaptionLimit is the max length (runes) of a media caption.
- telegramCaptionLimit = 1024
- // telegramMaxAttempts caps retries when the API returns 429.
- telegramMaxAttempts = 5
-)
-
-type TelegramTranscoder struct {
- botToken string
- channelID int
- groupID int
- lastUpdateID int
- imageBaseURL string
- previewBaseURL string
- previewExtension string
- stateDir string
- http *http.Client
-}
-
-func NewTelegramTranscoder(cfg any) (Transcoder, error) {
- tgCfg, ok := cfg.(*config.TelegramConfig)
- if !ok {
- return nil, fmt.Errorf("invalid config for telegram transcoder")
- }
-
- transcoder := &TelegramTranscoder{
- botToken: tgCfg.BotToken,
- channelID: tgCfg.ChannelID,
- groupID: tgCfg.GroupID,
- lastUpdateID: -1,
- imageBaseURL: strings.TrimRight(tgCfg.ImageBaseURL, "/"),
- previewBaseURL: strings.TrimRight(tgCfg.PreviewBaseURL, "/"),
- previewExtension: tgCfg.PreviewExtension,
- stateDir: tgCfg.StateDir,
- http: &http.Client{Timeout: time.Minute},
- }
-
- if _, err := transcoder.getUpdates(); err != nil {
- return nil, fmt.Errorf("failed to get last update id for checkpoint: %w", err)
- }
-
- return transcoder, nil
-}
-
-func (t *TelegramTranscoder) Name() string { return "telegram" }
-
-func (t *TelegramTranscoder) Finalize() error { return nil }
-
-// telegramState records, per draft, which section indices have already been
-// posted and the message ids they produced. Posting is append-only: a section
-// already present here is never re-sent.
-type telegramState struct {
- ChannelID string `json:"channelID"`
- Posted map[string][]int `json:"posted"`
-}
-
-func (t *TelegramTranscoder) Transcode(doc *draft.Document) error {
- statePath := filepath.Join(t.stateDir, doc.Codename+".telegram-state.json")
- state, err := loadTelegramState(statePath)
- if err != nil {
- return fmt.Errorf("load telegram state %q: %w", statePath, err)
- }
-
- if state.ChannelID != "" && state.ChannelID != strconv.Itoa(t.channelID) {
- slog.Warn("telegram channel changed, re-posting all sections to the new channel",
- slog.String("codename", doc.Codename),
- slog.String("old", state.ChannelID),
- slog.Int("new", t.channelID))
- state.Posted = map[string][]int{}
- }
- state.ChannelID = strconv.Itoa(t.channelID)
-
- for _, section := range doc.Sections {
- idx := strconv.Itoa(section.Index)
- if _, done := state.Posted[idx]; done {
- continue
- }
-
- var msgIDs []int
- pendingText := ""
-
- // flushText sends any buffered text as its own message.
- flushText := func() error {
- if pendingText == "" {
- return nil
- }
- id, err := t.sendMessage(pendingText)
- if err != nil {
- return err
- }
- msgIDs = append(msgIDs, id)
- pendingText = ""
- return nil
- }
-
- for _, block := range section.Blocks {
- switch block.Kind {
- case draft.TextBlock:
- text := stripRules(block.Text)
- if text == "" {
- continue
- }
- // flush any earlier text before buffering this one
- if err := flushText(); err != nil {
- return fmt.Errorf("send section %d text: %w", section.Index, err)
- }
- pendingText = text
- case draft.GalleryBlock:
- // Attach the immediately preceding text as the album caption when
- // it fits; otherwise send it as its own message first.
- caption := ""
- if pendingText != "" {
- if len([]rune(pendingText)) <= telegramCaptionLimit {
- caption = pendingText
- pendingText = ""
- } else if err := flushText(); err != nil {
- return fmt.Errorf("send section %d text: %w", section.Index, err)
- }
- }
- for i, group := range splitPhotos(block.Gallery.Photos, telegramMediaGroupLimit) {
- groupCaption := ""
- if i == 0 {
- groupCaption = caption
- }
- ids, err := t.sendMediaGroup(group, groupCaption, "photo", t.channelID, 0)
- if err != nil {
- return fmt.Errorf("send section %d gallery: %w", section.Index, err)
- }
- msgIDs = append(msgIDs, ids...)
- if err := flushText(); err != nil {
- return fmt.Errorf("send section %d text: %w", section.Index, err)
- }
- if len(msgIDs) > 0 {
- time.Sleep(5 * time.Second)
- forwardedID, err := t.getForwardedID(msgIDs)
- if err != nil {
- return fmt.Errorf("send full documents for section %d gallery: %w", section.Index, err)
- }
- if forwardedID == 0 {
- return fmt.Errorf("fail to get forwarded id of section %d gallery", section.Index)
- }
- _, err = t.sendMediaGroup(group, "", "document", t.groupID, forwardedID)
- if err != nil {
- return fmt.Errorf("send section %d documents: %w", section.Index, err)
- }
- }
- }
- }
- }
-
- state.Posted[idx] = msgIDs
- if err := saveTelegramState(statePath, state); err != nil {
- return fmt.Errorf("save telegram state %q: %w", statePath, err)
- }
- slog.Info("telegram posted section",
- slog.String("codename", doc.Codename),
- slog.Int("section", section.Index),
- slog.Int("messages", len(msgIDs)))
- }
-
- return nil
-}
-
-func loadTelegramState(path string) (*telegramState, error) {
- raw, err := os.ReadFile(path)
- if err != nil {
- if os.IsNotExist(err) {
- return &telegramState{Posted: map[string][]int{}}, nil
- }
- return nil, err
- }
- var state telegramState
- if err := json.Unmarshal(raw, &state); err != nil {
- return nil, err
- }
- if state.Posted == nil {
- state.Posted = map[string][]int{}
- }
- return &state, nil
-}
-
-func saveTelegramState(path string, state *telegramState) error {
- if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
- return err
- }
- raw, err := json.MarshalIndent(state, "", " ")
- if err != nil {
- return err
- }
- return os.WriteFile(path, raw, 0o644)
-}
-
-// stripRules drops standalone "---" horizontal-rule lines (meaningless in a
-// Telegram message) and trims the remaining text.
-func stripRules(text string) string {
- lines := strings.Split(text, "\n")
- kept := lines[:0]
- for _, line := range lines {
- if strings.TrimSpace(line) == "---" {
- continue
- }
- kept = append(kept, line)
- }
- return strings.TrimSpace(strings.Join(kept, "\n"))
-}
-
-// splitPhotos divides photos into near-equal groups each no larger than limit.
-func splitPhotos(photos []draft.Photo, limit int) [][]draft.Photo {
- n := len(photos)
- if n == 0 {
- return nil
- }
- groupCount := (n + limit - 1) / limit
- base := n / groupCount
- rem := n % groupCount
-
- var groups [][]draft.Photo
- start := 0
- for g := range groupCount {
- size := base
- if g < rem {
- size++
- }
- groups = append(groups, photos[start:start+size])
- start += size
- }
- return groups
-}
-
-type inputMediaPhoto struct {
- Type string `json:"type"`
- Media string `json:"media"`
- Caption string `json:"caption,omitempty"`
- ParseMode string `json:"parse_mode,omitempty"`
-}
-
-func (t *TelegramTranscoder) photoURL(key string) string {
- return t.imageBaseURL + "/" + key
-}
-
-func (t *TelegramTranscoder) previewURL(key string) string {
- base := t.previewBaseURL
- if base == "" {
- base = t.imageBaseURL
- }
- if t.previewExtension != "" {
- key = strings.TrimSuffix(key, filepath.Ext(key)) + t.previewExtension
- }
- return base + "/" + key
-}
-
-func (t *TelegramTranscoder) sendMessage(text string) (int, error) {
- payload := map[string]any{
- "chat_id": strconv.Itoa(t.channelID),
- "text": markdownToTelegramHTML(text),
- "parse_mode": "HTML",
- "disable_web_page_preview": true,
- }
- var result struct {
- MessageID int `json:"message_id"`
- }
- if err := t.call("sendMessage", payload, &result); err != nil {
- return 0, err
- }
- return result.MessageID, nil
-}
-
-func (t *TelegramTranscoder) sendMediaGroup(photos []draft.Photo, caption string, sendAsType string, id int, replyTo int) ([]int, error) {
- media := make([]inputMediaPhoto, 0, len(photos))
- files := map[string]uploadFile{}
- for i, p := range photos {
- item := inputMediaPhoto{Type: sendAsType}
- if sendAsType == "document" {
- // Upload the original file directly rather than passing Telegram a
- // URL to fetch: full-resolution originals routinely fail Telegram's
- // server-side downloader with WEBPAGE_CURL_FAILED.
- data, err := t.downloadPhoto(p.Key)
- if err != nil {
- return nil, fmt.Errorf("download %q for upload: %w", p.Key, err)
- }
- field := fmt.Sprintf("file%d", i)
- files[field] = uploadFile{name: filepath.Base(p.Key), data: data}
- item.Media = "attach://" + field
- } else {
- item.Media = t.previewURL(p.Key)
- }
- // Telegram shows an album's caption in the feed only when exactly one
- // item is captioned. So the section text goes on the first photo and
- // every other photo is left uncaptioned (per-photo captions dropped).
- if i == 0 && caption != "" {
- item.Caption = markdownToTelegramHTML(caption)
- item.ParseMode = "HTML"
- }
- media = append(media, item)
- }
-
- var result []struct {
- MessageID int `json:"message_id"`
- }
-
- if len(files) > 0 {
- mediaJSON, err := json.Marshal(media)
- if err != nil {
- return nil, err
- }
- fields := map[string]string{
- "chat_id": strconv.Itoa(id),
- "media": string(mediaJSON),
- }
- if replyTo != 0 {
- replyJSON, err := json.Marshal(map[string]any{"message_id": replyTo})
- if err != nil {
- return nil, err
- }
- fields["reply_parameters"] = string(replyJSON)
- }
- if err := t.callMultipart("sendMediaGroup", fields, files, &result); err != nil {
- return nil, err
- }
- } else {
- payload := map[string]any{
- "chat_id": strconv.Itoa(id),
- "media": media,
- }
- if replyTo != 0 {
- payload["reply_parameters"] = map[string]any{
- "message_id": replyTo,
- }
- }
- if err := t.call("sendMediaGroup", payload, &result); err != nil {
- return nil, err
- }
- }
-
- ids := make([]int, 0, len(result))
- for _, r := range result {
- ids = append(ids, r.MessageID)
- }
- return ids, nil
-}
-
-type uploadFile struct {
- name string
- data []byte
-}
-
-// downloadPhoto fetches the original bytes for a photo key so they can be
-// uploaded directly to Telegram instead of fetched server-side by Telegram.
-func (t *TelegramTranscoder) downloadPhoto(key string) ([]byte, error) {
- url := t.photoURL(key)
- resp, err := t.http.Get(url)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("unexpected status %d fetching %q", resp.StatusCode, url)
- }
- return io.ReadAll(resp.Body)
-}
-
-func (t *TelegramTranscoder) getForwardedID(channelMessageIDs []int) (int, error) {
- var payloads []getUpdatesPayload
- var err error
- slog.Debug("search for forwarded message id", slog.Any("original_id", channelMessageIDs), slog.Int("group_id", t.groupID), slog.Int("channel_id", t.channelID))
-
- for range 3 {
- for payloads, err = t.getUpdates(); len(payloads) > 0 && err == nil; payloads, err = t.getUpdates() {
- for _, payload := range payloads {
- if payload.Message.MessageOrigin.Type == "channel" &&
- payload.Message.MessageOrigin.Chat.ID == t.channelID &&
- slices.Contains(channelMessageIDs, payload.Message.MessageOrigin.MessageID) &&
- payload.Message.Chat.ID == t.groupID {
- return payload.Message.MessageID, nil
- }
- }
- }
- time.Sleep(3 * time.Second)
- }
-
- return 0, err
-}
-
-type getUpdatesPayload struct {
- UpdateID int `json:"update_id"`
- Message struct {
- MessageID int `json:"message_id"`
- Chat struct {
- ID int `json:"id"`
- Title string `json:"title"`
- Type string `json:"type"`
- } `json:"chat"`
- MessageOrigin struct {
- Type string `json:"type"`
- Date int `json:"date"`
- Chat struct {
- ID int `json:"id"`
- Title string `json:"title"`
- Type string `json:"type"`
- } `json:"chat"`
- MessageID int `json:"message_id"`
- } `json:"forward_origin"`
- Date int `json:"date"`
- Text string `json:"text"`
- } `json:"message"`
-}
-
-func (t *TelegramTranscoder) getUpdates() ([]getUpdatesPayload, error) {
- var out []getUpdatesPayload
-
- payload := struct {
- Offset int `json:"offset"`
- }{
- Offset: t.lastUpdateID,
- }
-
- if err := t.call("getUpdates", payload, &out); err != nil {
- return nil, fmt.Errorf("failed to transcode received message: %w", err)
- }
-
- for _, p := range out {
- pjson, _ := json.Marshal(p)
- slog.Debug("parsed new update", slog.String("payload", string(pjson)))
- if p.UpdateID > t.lastUpdateID {
- t.lastUpdateID = p.UpdateID
- }
- }
- t.lastUpdateID++
-
- return out, nil
-}
-
-// handleResponse decodes a Telegram API response envelope. When the API asks us
-// to back off (HTTP 429) it returns a positive wait duration and a nil error so
-// the caller can retry; otherwise it unmarshals the result into out (if any) or
-// returns the API error verbatim.
-func (t *TelegramTranscoder) handleResponse(method string, resp *http.Response, attempt int, out any) (time.Duration, error) {
- var envelope struct {
- OK bool `json:"ok"`
- ErrorCode int `json:"error_code"`
- Description string `json:"description"`
- Parameters struct {
- RetryAfter int `json:"retry_after"`
- } `json:"parameters"`
- Result json.RawMessage `json:"result"`
- }
- decodeErr := json.NewDecoder(resp.Body).Decode(&envelope)
- resp.Body.Close()
- if decodeErr != nil {
- return 0, fmt.Errorf("decode telegram response (%s): %w", method, decodeErr)
- }
-
- envelopeBytes, _ := json.Marshal(envelope)
- slog.Debug("telegram response", slog.String("method", method), slog.String("response", string(envelopeBytes)))
-
- if envelope.OK {
- if out == nil {
- return 0, nil
- }
- return 0, json.Unmarshal(envelope.Result, out)
- }
-
- // 429 Too Many Requests: Telegram tells us how long to wait in
- // parameters.retry_after. Back off and retry instead of failing.
- if envelope.ErrorCode == 429 && attempt < telegramMaxAttempts {
- wait := time.Duration(envelope.Parameters.RetryAfter) * time.Second
- if wait <= 0 {
- wait = time.Second
- }
- slog.Warn("telegram rate limited, backing off",
- slog.String("method", method),
- slog.Int("attempt", attempt),
- slog.Duration("retry_after", wait))
- return wait, nil
- }
-
- return 0, fmt.Errorf("telegram %s failed: %s", method, envelope.Description)
-}
-
-// call invokes a Telegram Bot API method with a JSON body and decodes the
-// "result" field into out. The API's error description is surfaced verbatim.
-func (t *TelegramTranscoder) call(method string, payload any, out any) error {
- body, err := json.Marshal(payload)
- if err != nil {
- return err
- }
-
- url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", t.botToken, method)
-
- for attempt := 1; ; attempt++ {
- req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
- if err != nil {
- return err
- }
- req.Header.Set("Content-Type", "application/json")
-
- slog.Debug("send message", slog.String("method", method), slog.String("request", string(body)))
-
- resp, err := t.http.Do(req)
- if err != nil {
- return err
- }
-
- wait, err := t.handleResponse(method, resp, attempt, out)
- if wait > 0 {
- time.Sleep(wait)
- continue
- }
- return err
- }
-}
-
-// callMultipart invokes a Telegram Bot API method using multipart/form-data,
-// uploading the given files directly instead of handing Telegram URLs to fetch
-// server-side. fields carries the non-file form values (chat_id, media JSON,
-// reply_parameters, ...); files maps a form field name to its bytes and upload
-// filename, referenced from the media JSON via attach://<field>.
-func (t *TelegramTranscoder) callMultipart(method string, fields map[string]string, files map[string]uploadFile, out any) error {
- url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", t.botToken, method)
-
- for attempt := 1; ; attempt++ {
- var body bytes.Buffer
- w := multipart.NewWriter(&body)
- for k, v := range fields {
- if err := w.WriteField(k, v); err != nil {
- return err
- }
- }
- for field, f := range files {
- part, err := w.CreateFormFile(field, f.name)
- if err != nil {
- return err
- }
- if _, err := part.Write(f.data); err != nil {
- return err
- }
- }
- if err := w.Close(); err != nil {
- return err
- }
-
- req, err := http.NewRequest(http.MethodPost, url, &body)
- if err != nil {
- return err
- }
- req.Header.Set("Content-Type", w.FormDataContentType())
-
- slog.Debug("send media group upload", slog.String("method", method), slog.Int("files", len(files)))
-
- resp, err := t.http.Do(req)
- if err != nil {
- return err
- }
-
- wait, err := t.handleResponse(method, resp, attempt, out)
- if wait > 0 {
- time.Sleep(wait)
- continue
- }
- return err
- }
-}
diff --git a/internal/transcoder/transcoder.go b/internal/transcoder/transcoder.go
deleted file mode 100644
index 3dbdf7e..0000000
--- a/internal/transcoder/transcoder.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package transcoder
-
-import (
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft"
-)
-
-// Transcoder turns a parsed draft into a published artifact (an S3 page, a
-// Telegram thread, ...). Transcode runs once per draft; Finalize runs once
-// after all drafts have been processed.
-type Transcoder interface {
- Name() string
- Transcode(doc *draft.Document) error
- Finalize() error
-}
-
-var NewTranscoderMap = map[string]func(cfg any) (Transcoder, error){
- "sayauz": NewSayauzTranscoder,
- "telegram": NewTelegramTranscoder,
-}
diff --git a/main.go b/main.go
index 6648aeb..7c50824 100644
--- a/main.go
+++ b/main.go
@@ -2,16 +2,13 @@ package main
import (
"flag"
- "io/fs"
"log/slog"
"os"
- "path/filepath"
- "strings"
"sync"
"github.com/SayaAndy/saya-today-article-metadata-add/config"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/transcoder"
+ "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
+ "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage"
)
var configPath = flag.String("c", "config.json", "Path to the configuration file")
@@ -19,8 +16,8 @@ var configPath = flag.String("c", "config.json", "Path to the configuration file
func main() {
flag.Parse()
- cfg, err := config.InitConfig(*configPath)
- if err != nil {
+ cfg := &config.Config{}
+ if err := config.LoadConfig(*configPath, cfg); err != nil {
slog.Error("fail to load configuration", slog.String("error", err.Error()))
os.Exit(1)
}
@@ -28,85 +25,73 @@ func main() {
slog.SetLogLoggerLevel(cfg.LogLevel)
slog.Info("starting metadata extractor...")
- transcoders := make([]transcoder.Transcoder, 0, len(cfg.Transcoders))
- for _, tcCfg := range cfg.Transcoders {
- newTranscoder, ok := transcoder.NewTranscoderMap[tcCfg.Type]
- if !ok {
- slog.Error("unsupported transcoder type", slog.String("type", tcCfg.Type))
- os.Exit(1)
- }
- t, err := newTranscoder(tcCfg.Config)
- if err != nil {
- slog.Error("fail to initialize transcoder", slog.String("type", tcCfg.Type), slog.String("error", err.Error()))
- os.Exit(1)
- }
- transcoders = append(transcoders, t)
- slog.Info("initialized transcoder", slog.String("type", tcCfg.Type))
+ storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage)
+ if err != nil {
+ slog.Error("fail to initialize input client", slog.String("error", err.Error()))
+ os.Exit(1)
}
- var drafts []string
- err = filepath.WalkDir(cfg.DraftDir, func(path string, d fs.DirEntry, err error) error {
- if err != nil {
- return err
- }
- if d.IsDir() {
- return nil
- }
- if strings.HasSuffix(path, cfg.DraftSuffix) {
- drafts = append(drafts, path)
- }
- return nil
- })
+ generalLogger := slog.With(
+ slog.String("storage_type", cfg.Storage.Type),
+ )
+ generalLogger.Info("initialized storage client")
+
+ files, err := storageClient.Scan()
if err != nil {
- slog.Error("fail to scan draft directory", slog.String("dir", cfg.DraftDir), slog.String("error", err.Error()))
+ generalLogger.Error("fail to scan input files", slog.String("error", err.Error()))
os.Exit(1)
}
- slog.Info("scanned drafts", slog.Int("draft_count", len(drafts)))
+ generalLogger.Info("scanned files", slog.Int("file_count", len(files)))
semaphore := make(chan struct{}, cfg.MaxConcurrentJobs)
var wg sync.WaitGroup
- wg.Add(len(drafts))
+ wg.Add(len(files))
- for _, draftPath := range drafts {
+ for i, file := range files {
semaphore <- struct{}{}
- go func(path string) {
+ go func(index int, inputName string) {
defer wg.Done()
defer func() { <-semaphore }()
+ if !storageClient.FileHasChanged(file) {
+ generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file))
+ return
+ }
+ generalLogger.Debug("processing a file", slog.String("file", file))
- codename := strings.TrimSuffix(filepath.Base(path), cfg.DraftSuffix)
- fileLogger := slog.With(slog.String("draft", path), slog.String("codename", codename))
+ reader, sz, err := storageClient.GetReader(file)
+ if err != nil {
+ generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error()))
+ return
+ }
+ defer reader.Close()
- content, err := os.ReadFile(path)
+ content := make([]byte, sz)
+ ln, err := reader.Read(content)
if err != nil {
- fileLogger.Warn("fail to read draft", slog.String("error", err.Error()))
+ generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error()))
return
}
+ generalLogger.Debug("read content from a file",
+ slog.String("file", file),
+ slog.Int64("expected_size", sz),
+ slog.Int("output_size", ln))
- doc, err := draft.ParseDraft(path, codename, content)
+ metadata, _, err := frontmatter.ParseFrontmatter(content)
if err != nil {
- fileLogger.Warn("fail to parse draft", slog.String("error", err.Error()))
+ generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error()))
return
}
- if doc.Metadata == nil {
- fileLogger.Info("skip draft without frontmatter metadata")
+
+ if metadata == nil {
+ generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file))
return
}
- for _, t := range transcoders {
- if err := t.Transcode(doc); err != nil {
- fileLogger.Warn("transcoder failed", slog.String("transcoder", t.Name()), slog.String("error", err.Error()))
- }
+ if err = storageClient.WriteMetadata(file, metadata); err != nil {
+ generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error()))
}
- }(draftPath)
+ }(i, file)
}
wg.Wait()
-
- for _, t := range transcoders {
- if err := t.Finalize(); err != nil {
- slog.Error("fail to finalize transcoder", slog.String("transcoder", t.Name()), slog.String("error", err.Error()))
- os.Exit(1)
- }
- slog.Info("finalized transcoder", slog.String("transcoder", t.Name()))
- }
}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-26 17:54:01 +0000