summaryrefslogtreecommitdiff
path: root/internal
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat (limited to 'internal')
-rw-r--r--internal/draft/parser.go179
-rw-r--r--internal/draft/parser_test.go83
-rw-r--r--internal/storage/b2.go142
-rw-r--r--internal/storage/index.go92
-rw-r--r--internal/storage/s3.go377
-rw-r--r--internal/storage/storage_interface.go11
-rw-r--r--internal/transcoder/format_test.go28
-rw-r--r--internal/transcoder/markdown.go164
-rw-r--r--internal/transcoder/sayauz.go138
-rw-r--r--internal/transcoder/telegram.go584
-rw-r--r--internal/transcoder/transcoder.go19
11 files changed, 228 insertions, 1589 deletions
diff --git a/internal/draft/parser.go b/internal/draft/parser.go
deleted file mode 100644
index 2761c87..0000000
--- a/internal/draft/parser.go
+++ /dev/null
@@ -1,179 +0,0 @@
-package draft
-
-import (
- "regexp"
- "strings"
-
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
-)
-
-// layoutTokenRe matches gallery layout hints like "2x", "3x" that are
-// meaningful only to the sayauz renderer and must be dropped elsewhere.
-var layoutTokenRe = regexp.MustCompile(`^\d+x$`)
-
-type BlockKind int
-
-const (
- TextBlock BlockKind = iota
- GalleryBlock
-)
-
-// Block is one ordered piece of a section: either a run of markdown text or a
-// gallery. Exactly one of Text / Gallery is meaningful depending on Kind.
-type Block struct {
- Kind BlockKind
- Text string
- Gallery *Gallery
-}
-
-type Gallery struct {
- Timezone string
- KeyPrefix string
- Photos []Photo
-}
-
-type Photo struct {
- Key string // KeyPrefix + filename, e.g. "Sevsk/20241008-093833.jpg"
- Caption string
-}
-
-// Section is the content between two "..." separator lines, split into ordered
-// text and gallery blocks.
-type Section struct {
- Index int
- Blocks []Block
-}
-
-type Document struct {
- SourcePath string
- Codename string
- Metadata *frontmatter.Metadata
- RawContent []byte
- Body string
- Sections []Section
-}
-
-// ParseDraft parses a draft file: frontmatter metadata, the markdown body, and
-// the body split into sections (on "..." lines) of ordered text/gallery blocks.
-func ParseDraft(sourcePath, codename string, content []byte) (*Document, error) {
- metadata, body, err := frontmatter.ParseFrontmatter(content)
- if err != nil {
- return nil, err
- }
-
- doc := &Document{
- SourcePath: sourcePath,
- Codename: codename,
- Metadata: metadata,
- RawContent: content,
- Body: string(body),
- }
- doc.Sections = splitSections(doc.Body)
-
- return doc, nil
-}
-
-func splitSections(body string) []Section {
- lines := strings.Split(body, "\n")
-
- var sections []Section
- var chunk []string
-
- flush := func() {
- blocks := parseBlocks(chunk)
- chunk = nil
- if len(blocks) == 0 {
- return
- }
- sections = append(sections, Section{Index: len(sections), Blocks: blocks})
- }
-
- for _, line := range lines {
- if strings.TrimSpace(line) == "..." {
- flush()
- continue
- }
- chunk = append(chunk, line)
- }
- flush()
-
- return sections
-}
-
-func parseBlocks(lines []string) []Block {
- var blocks []Block
- var text []string
-
- flushText := func() {
- joined := strings.TrimSpace(strings.Join(text, "\n"))
- text = nil
- if joined != "" {
- blocks = append(blocks, Block{Kind: TextBlock, Text: joined})
- }
- }
-
- for i := 0; i < len(lines); i++ {
- trimmed := strings.TrimSpace(lines[i])
- if strings.HasPrefix(trimmed, "{Gallery:") {
- flushText()
- gallery, next := parseGallery(lines, i)
- if gallery != nil {
- blocks = append(blocks, Block{Kind: GalleryBlock, Gallery: gallery})
- }
- i = next
- continue
- }
- text = append(text, lines[i])
- }
- flushText()
-
- return blocks
-}
-
-// parseGallery reads a {Gallery:...} block starting at lines[start] and returns
-// the parsed gallery plus the index of the closing {/Gallery} line (or the last
-// consumed line if unterminated).
-func parseGallery(lines []string, start int) (*Gallery, int) {
- header := strings.TrimSpace(lines[start])
- header = strings.TrimPrefix(header, "{Gallery:")
- header = strings.TrimSuffix(header, "}")
- tz, keyPrefix, _ := strings.Cut(header, ":")
-
- gallery := &Gallery{Timezone: tz, KeyPrefix: keyPrefix}
-
- i := start + 1
- for ; i < len(lines); i++ {
- trimmed := strings.TrimSpace(lines[i])
- if trimmed == "{/Gallery}" {
- break
- }
- if trimmed == "" {
- continue
- }
- gallery.Photos = append(gallery.Photos, parsePhoto(keyPrefix, trimmed))
- }
-
- return gallery, i
-}
-
-func parsePhoto(keyPrefix, line string) Photo {
- parts := strings.Split(line, "|")
- for i := range parts {
- parts[i] = strings.TrimSpace(parts[i])
- }
-
- filename := parts[0]
-
- var captionParts []string
- for _, p := range parts[1:] {
- if p == "" || layoutTokenRe.MatchString(p) {
- continue
- }
- captionParts = append(captionParts, p)
- }
-
- return Photo{
- Key: keyPrefix + filename,
- Caption: strings.Join(captionParts, " "),
- }
-}
diff --git a/internal/draft/parser_test.go b/internal/draft/parser_test.go
deleted file mode 100644
index c6d454d..0000000
--- a/internal/draft/parser_test.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package draft
-
-import (
- "os"
- "strings"
- "testing"
-)
-
-func TestParseDraftSevsk(t *testing.T) {
- raw, err := os.ReadFile("../../.draft/sevsk.md.draft")
- if err != nil {
- t.Fatal(err)
- }
-
- doc, err := ParseDraft("../../.draft/sevsk.md.draft", "sevsk", raw)
- if err != nil {
- t.Fatal(err)
- }
- if doc.Metadata == nil || doc.Metadata.Title == "" {
- t.Fatal("expected frontmatter metadata")
- }
-
- var galleries int
- for _, section := range doc.Sections {
- for _, block := range section.Blocks {
- if block.Kind != GalleryBlock {
- continue
- }
- galleries++
- for _, p := range block.Gallery.Photos {
- if !strings.HasPrefix(p.Key, "Sevsk/20241008-") {
- t.Errorf("unexpected photo key %q", p.Key)
- }
- if strings.Contains(p.Caption, "2x") {
- t.Errorf("layout token leaked into caption %q", p.Caption)
- }
- }
- }
- }
- if galleries == 0 {
- t.Fatal("expected at least one gallery")
- }
-
- // First gallery: 4 photos, captions on photos 1 and 3, none on 2; photo 4 had "2x |" => empty caption.
- first := firstGallery(doc)
- if first == nil || len(first.Photos) != 4 {
- t.Fatalf("first gallery: want 4 photos, got %v", first)
- }
- if first.Timezone != "Europe/Moscow" {
- t.Errorf("first gallery tz = %q", first.Timezone)
- }
- if first.Photos[0].Caption != "Никольская церковь. Действующая" {
- t.Errorf("photo[0] caption = %q", first.Photos[0].Caption)
- }
- if first.Photos[1].Caption != "" {
- t.Errorf("photo[1] caption = %q, want empty", first.Photos[1].Caption)
- }
- if first.Photos[3].Caption != "" {
- t.Errorf("photo[3] caption = %q, want empty (only 2x token)", first.Photos[3].Caption)
- }
- if first.Photos[3].Key != "Sevsk/20241008-094036.jpg" {
- t.Errorf("photo[3] key = %q", first.Photos[3].Key)
- }
-}
-
-func firstGallery(doc *Document) *Gallery {
- for _, section := range doc.Sections {
- for _, block := range section.Blocks {
- if block.Kind == GalleryBlock {
- return block.Gallery
- }
- }
- }
- return nil
-}
-
-func TestSplitSectionsSeparator(t *testing.T) {
- body := "intro\n...\nmiddle\n...\nend"
- sections := splitSections(body)
- if len(sections) != 3 {
- t.Fatalf("want 3 sections, got %d", len(sections))
- }
-}
diff --git a/internal/storage/b2.go b/internal/storage/b2.go
index 6697023..b02261f 100644
--- a/internal/storage/b2.go
+++ b/internal/storage/b2.go
@@ -3,6 +3,7 @@ package storage
import (
"context"
"fmt"
+ "io"
"strconv"
"strings"
"time"
@@ -15,12 +16,13 @@ import (
var _ StorageClient = &B2StorageClient{}
type B2StorageClient struct {
- prefix string
- bucket *b2.Bucket
- b2cl *b2.Client
+ prefix string
+ bucket *b2.Bucket
+ b2cl *b2.Client
+ draftModeCfg *config.DraftModeConfig
}
-func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
+func NewB2StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftModeConfig) (StorageClient, error) {
if cfg.Type != "b2" {
return nil, fmt.Errorf("invalid storage type for B2InputClient")
}
@@ -36,25 +38,73 @@ func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
return nil, err
}
- return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
+ draftModeCfgCopy := *draftModeCfg
+
+ return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, draftModeCfg: &draftModeCfgCopy}, nil
}
-func (sc *B2StorageClient) GetMetadata(key string) (map[string]string, error) {
- obj := sc.bucket.Object(sc.prefix + key)
+func (sc *B2StorageClient) Scan() ([]string, error) {
+ filePaths := []string{}
+
+ iter := sc.bucket.List(context.Background(), b2.ListPrefix(sc.prefix))
+
+ for iter.Next() {
+ obj := iter.Object()
+ if obj == nil {
+ return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ }
+
+ attrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("get attributes for object: %w", err)
+ }
+
+ if attrs.Status != b2.Uploaded {
+ continue
+ }
+
+ name := obj.Name()
+ if !strings.HasSuffix(name, ".md") {
+ continue
+ }
+
+ if sc.draftModeCfg.Enabled && !strings.HasSuffix(name, sc.draftModeCfg.DraftSuffix) {
+ continue
+ }
+
+ filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix))
+ }
+
+ if err := iter.Err(); err != nil {
+ return nil, fmt.Errorf("iterate over B2 objects: %w", err)
+ }
+
+ return filePaths, nil
+}
+
+func (sc *B2StorageClient) GetReader(path string) (io.ReadCloser, int64, error) {
+ obj := sc.bucket.Object(sc.prefix + path)
if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ return nil, 0, fmt.Errorf("failed to reference object in B2 bucket")
}
attrs, err := obj.Attrs(context.Background())
if err != nil {
- if b2.IsNotExist(err) {
- return map[string]string{}, nil
- }
- return nil, fmt.Errorf("get attributes of B2 object %q: %w", sc.prefix+key, err)
+ return nil, 0, fmt.Errorf("error getting attributes of an object: %w", err)
}
- return attrs.Info, nil
+
+ return obj.NewReader(context.Background()), attrs.Size, nil
}
-func (sc *B2StorageClient) Put(key string, content []byte, metadata *frontmatter.Metadata) error {
+func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error {
+ draft := sc.bucket.Object(sc.prefix + path)
+ if draft == nil {
+ return fmt.Errorf("failed to reference draft object in B2 bucket")
+ }
+ draftAttrs, err := draft.Attrs(context.Background())
+ if err != nil {
+ return fmt.Errorf("error getting attributes of a draft object: %w", err)
+ }
+
geolocationParts := strings.Split(metadata.Geolocation, " ")
if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 {
return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string")
@@ -81,19 +131,32 @@ func (sc *B2StorageClient) Put(key string, content []byte, metadata *frontmatter
attrs := &b2.Attrs{
ContentType: "text/markdown; charset=utf-8",
Info: map[string]string{
- "title": metadata.Title,
- "short-description": metadata.ShortDescription,
- "action-date": metadata.ActionDate,
- "published-time": metadata.PublishedTime.Format(time.RFC3339),
- "thumbnail": metadata.Thumbnail,
- "tags": strings.Join(metadata.Tags, ","),
- "geolocation": metadata.Geolocation,
- "medley": medley,
+ "title": metadata.Title,
+ "short-description": metadata.ShortDescription,
+ "action-date": metadata.ActionDate,
+ "published-time": metadata.PublishedTime.Format(time.RFC3339),
+ "thumbnail": metadata.Thumbnail,
+ "tags": strings.Join(metadata.Tags, ","),
+ "geolocation": metadata.Geolocation,
+ "medley": medley,
+ "metadata-last-update-sha1": draftAttrs.SHA1,
}}
- prod := sc.bucket.Object(sc.prefix + key)
- if prod == nil {
- return fmt.Errorf("failed to reference prod object in B2 bucket")
+ reader := draft.NewReader(context.Background())
+ content := make([]byte, draftAttrs.Size)
+ if _, err = reader.Read(content); err != nil {
+ return fmt.Errorf("failed to read a draft object back for writing (required for attribute setting): %w", err)
+ }
+
+ var prod *b2.Object
+ if sc.draftModeCfg.Enabled {
+ prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+ prod = sc.bucket.Object(sc.prefix + prodPath)
+ if prod == nil {
+ return fmt.Errorf("failed to reference prod object in B2 bucket")
+ }
+ } else {
+ prod = draft
}
writer := prod.NewWriter(context.Background(), b2.WithAttrsOption(attrs))
@@ -105,6 +168,31 @@ func (sc *B2StorageClient) Put(key string, content []byte, metadata *frontmatter
return nil
}
-func (sc *B2StorageClient) BuildIndex() error {
- return fmt.Errorf("BuildIndex not implemented for B2 storage")
+func (sc *B2StorageClient) CompareDraftAndProd(path string) (changed bool) {
+ draft := sc.bucket.Object(sc.prefix + path)
+ if draft == nil {
+ return false
+ }
+
+ prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+ prod := sc.bucket.Object(sc.prefix + prodPath)
+ if prod == nil {
+ return true
+ }
+
+ draftAttrs, err := draft.Attrs(context.Background())
+ if err != nil {
+ return false
+ }
+ prodAttrs, err := prod.Attrs(context.Background())
+ if err != nil {
+ return true
+ }
+
+ lastUpdateSha1, ok := prodAttrs.Info["metadata-last-update-sha1"]
+ if !ok {
+ return true
+ }
+
+ return draftAttrs.SHA1 != lastUpdateSha1
}
diff --git a/internal/storage/index.go b/internal/storage/index.go
deleted file mode 100644
index bfeafbe..0000000
--- a/internal/storage/index.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package storage
-
-import (
- "encoding/json"
- "fmt"
- "time"
-)
-
-const IndexFileName = "index.json"
-const MedleysIndexFileName = "medleys.json"
-
-const IndexSchemaVersion = 2
-
-type IndexEntry struct {
- Link string `json:"link"`
- ModifiedTime time.Time `json:"modifiedTime"`
- Title string `json:"title"`
- ShortDescription string `json:"shortDescription"`
- ActionDate string `json:"actionDate"`
- PublishedTime time.Time `json:"publishedTime"`
- Thumbnail string `json:"thumbnail"`
- Tags []string `json:"tags"`
- Geolocation string `json:"geolocation"`
- Medley string `json:"medley,omitempty"`
- MedleyPart int `json:"medleyPart,omitempty"`
-}
-
-type IndexV2Category struct {
- GeneratedAt time.Time `json:"generatedAt"`
- Pages map[string]IndexEntry `json:"pages"`
-}
-
-type IndexV1Category struct {
- GeneratedAt time.Time `json:"generatedAt"`
- Pages []IndexEntry `json:"pages"`
-}
-
-type Index struct {
- SchemaVersion int `json:"schemaVersion"`
- GeneratedAt time.Time `json:"generatedAt"`
- Categories any `json:"categories"`
-}
-
-func (idx *Index) UnmarshalJSON(data []byte) error {
- var tmp struct {
- SchemaVersion int `json:"schemaVersion"`
- GeneratedAt time.Time `json:"generatedAt"`
- Categories json.RawMessage `json:"categories"`
- }
-
- if err := json.Unmarshal(data, &tmp); err != nil {
- return err
- }
-
- idx.SchemaVersion = tmp.SchemaVersion
- idx.GeneratedAt = tmp.GeneratedAt
-
- switch tmp.SchemaVersion {
- case 1:
- var categories map[string]*IndexV1Category
- if err := json.Unmarshal(tmp.Categories, &categories); err != nil {
- return fmt.Errorf("unmarshal map[string]*IndexV1Category: %w", err)
- }
- idx.Categories = &categories
- case 2:
- var categories map[string]*IndexV2Category
- if err := json.Unmarshal(tmp.Categories, &categories); err != nil {
- return fmt.Errorf("unmarshal map[string]*IndexV2Category: %w", err)
- }
- idx.Categories = &categories
- default:
- return fmt.Errorf("unsupported index version: %d", tmp.SchemaVersion)
- }
-
- return nil
-}
-
-type IndexV1 struct {
- SchemaVersion int `json:"schemaVersion"`
- GeneratedAt time.Time `json:"generatedAt"`
- Categories map[string]*IndexV1Category `json:"categories"`
-}
-
-type MedleyEntry struct {
- Codename string `json:"codename"`
- Content []string `json:"content"`
-}
-
-type MedleyPageEntry struct {
- Codename string `json:"codename"`
- Position int `json:"position"`
-}
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index 5f3acc7..5e9bcd2 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -3,40 +3,31 @@ package storage
import (
"bytes"
"context"
- "encoding/json"
- "errors"
"fmt"
"io"
- "log/slog"
- "maps"
- "net/url"
- "slices"
"strconv"
"strings"
- "sync"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
- s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/SayaAndy/saya-today-article-metadata-add/config"
"github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
)
-const s3IndexConcurrency = 32
-
var _ StorageClient = &S3StorageClient{}
type S3StorageClient struct {
- prefix string
- bucket string
- client *s3.Client
+ prefix string
+ bucket string
+ client *s3.Client
+ draftModeCfg *config.DraftModeConfig
}
-func NewS3StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
+func NewS3StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftModeConfig) (StorageClient, error) {
if cfg.Type != "s3" {
return nil, fmt.Errorf("invalid storage type for S3StorageClient")
}
@@ -68,322 +59,166 @@ func NewS3StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
client := s3.NewFromConfig(awsCfg, s3Opts...)
+ draftModeCfgCopy := *draftModeCfg
+
return &S3StorageClient{
- client: client,
- bucket: s3cfg.BucketName,
- prefix: s3cfg.Prefix,
+ client: client,
+ bucket: s3cfg.BucketName,
+ prefix: s3cfg.Prefix,
+ draftModeCfg: &draftModeCfgCopy,
}, nil
}
-func (sc *S3StorageClient) GetMetadata(key string) (map[string]string, error) {
- head, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(sc.prefix + key),
- })
- if err != nil {
- var nsk *s3types.NoSuchKey
- var nf *s3types.NotFound
- if errors.As(err, &nsk) || errors.As(err, &nf) {
- return map[string]string{}, nil
- }
- return nil, fmt.Errorf("head S3 object %q: %w", sc.prefix+key, err)
- }
- return head.Metadata, nil
-}
-
-func (sc *S3StorageClient) Put(key string, content []byte, metadata *frontmatter.Metadata) error {
- geolocationParts := strings.Split(metadata.Geolocation, " ")
- if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 {
- return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string")
- }
- if len(geolocationParts) >= 2 {
- if _, err := strconv.ParseFloat(geolocationParts[0], 64); err != nil {
- return fmt.Errorf("invalid geolocation parameter, expected float for X: %w", err)
- }
- if _, err := strconv.ParseFloat(geolocationParts[1], 64); err != nil {
- return fmt.Errorf("invalid geolocation parameter, expected float for Y: %w", err)
- }
- }
- if len(geolocationParts) == 3 {
- if _, err := strconv.ParseFloat(geolocationParts[2], 64); err != nil {
- return fmt.Errorf("invalid geolocation parameter, expected float for area error: %w", err)
- }
- }
-
- s3Metadata := map[string]string{
- "title": url.QueryEscape(metadata.Title),
- "short-description": url.QueryEscape(metadata.ShortDescription),
- "action-date": metadata.ActionDate,
- "published-time": metadata.PublishedTime.Format(time.RFC3339),
- "thumbnail": url.QueryEscape(metadata.Thumbnail),
- "tags": strings.Join(metadata.Tags, ","),
- "geolocation": metadata.Geolocation,
- }
-
- targetKey := sc.prefix + key
- _, err := sc.client.PutObject(context.Background(), &s3.PutObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(targetKey),
- Body: bytes.NewReader(content),
- ContentType: aws.String("text/markdown; charset=utf-8"),
- Metadata: s3Metadata,
- })
- if err != nil {
- return fmt.Errorf("put S3 object %q: %w", targetKey, err)
- }
-
- return nil
-}
-
-func (sc *S3StorageClient) BuildIndex() error {
- type candidate struct {
- key string
- codename string
- lastModified time.Time
- }
- var candidates []candidate
+func (sc *S3StorageClient) Scan() ([]string, error) {
+ var filePaths []string
paginator := s3.NewListObjectsV2Paginator(sc.client, &s3.ListObjectsV2Input{
Bucket: aws.String(sc.bucket),
Prefix: aws.String(sc.prefix),
})
+
for paginator.HasMorePages() {
page, err := paginator.NextPage(context.Background())
if err != nil {
- return fmt.Errorf("list S3 objects: %w", err)
+ return nil, fmt.Errorf("list S3 objects: %w", err)
}
+
for _, obj := range page.Contents {
- key := aws.ToString(obj.Key)
- if key == IndexFileName {
+ name := aws.ToString(obj.Key)
+ if !strings.HasSuffix(name, ".md") {
continue
}
- if !strings.HasSuffix(key, ".md") {
+
+ if sc.draftModeCfg.Enabled && !strings.HasSuffix(name, sc.draftModeCfg.DraftSuffix) {
continue
}
- codename := key[strings.LastIndex(key, "/")+1 : strings.LastIndex(key, ".")]
- candidates = append(candidates, candidate{key, codename, aws.ToTime(obj.LastModified)})
- }
- }
- medleys, err := sc.scanMedleys()
- if err != nil {
- slog.Warn("skipped reading medleys due to an error", slog.String("error", err.Error()))
- }
- pageToMedleyMap := make(map[string]MedleyPageEntry)
- for _, medley := range medleys {
- for i, page := range medley.Content {
- pageToMedleyMap[page] = MedleyPageEntry{medley.Codename, i}
+ filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix))
}
}
- type result struct {
- catKey string
- codename string
- entry IndexEntry
- }
- results := make([]*result, len(candidates))
- sem := make(chan struct{}, s3IndexConcurrency)
- var wg sync.WaitGroup
- var firstErr error
- var errMu sync.Mutex
-
- for i, cand := range candidates {
- sem <- struct{}{}
- wg.Add(1)
- go func() {
- defer wg.Done()
- defer func() { <-sem }()
-
- head, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(sc.bucket),
- Key: aws.String(cand.key),
- })
- if err != nil {
- errMu.Lock()
- if firstErr == nil {
- firstErr = fmt.Errorf("head S3 object %s: %w", cand.key, err)
- }
- errMu.Unlock()
- return
- }
-
- if head.ContentType == nil || !strings.Contains(*head.ContentType, "text/markdown") {
- return
- }
- meta := head.Metadata
- if meta["title"] == "" {
- return
- }
-
- publishedTime, err := time.Parse(time.RFC3339, meta["published-time"])
- if err != nil {
- slog.Warn("skip entry with bad published-time", slog.String("key", cand.key), slog.String("error", err.Error()))
- return
- }
+ return filePaths, nil
+}
- catKey := cand.key[:strings.Index(cand.key, "/")]
+func (sc *S3StorageClient) GetReader(path string) (io.ReadCloser, int64, error) {
+ key := sc.prefix + path
- tags := strings.Split(meta["tags"], ",")
- slices.Sort(tags)
+ out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{
+ Bucket: aws.String(sc.bucket),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, 0, fmt.Errorf("get S3 object %q: %w", key, err)
+ }
- title, _ := url.QueryUnescape(meta["title"])
- shortDescription, _ := url.QueryUnescape(meta["short-description"])
- thumbnail, _ := url.QueryUnescape(meta["thumbnail"])
+ return out.Body, aws.ToInt64(out.ContentLength), nil
+}
- medleyName, medleyPart := "", 0
- if medley, ok := pageToMedleyMap[cand.codename]; ok {
- medleyName, medleyPart = medley.Codename, medley.Position
- }
+func (sc *S3StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error {
+ draftKey := sc.prefix + path
- results[i] = &result{
- catKey: catKey,
- codename: cand.codename,
- entry: IndexEntry{
- Link: cand.key,
- ModifiedTime: cand.lastModified,
- Title: title,
- ShortDescription: shortDescription,
- ActionDate: meta["action-date"],
- PublishedTime: publishedTime,
- Thumbnail: thumbnail,
- Tags: tags,
- Geolocation: meta["geolocation"],
- Medley: medleyName,
- MedleyPart: medleyPart,
- },
- }
- }()
+ getOut, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{
+ Bucket: aws.String(sc.bucket),
+ Key: aws.String(draftKey),
+ })
+ if err != nil {
+ return fmt.Errorf("get draft object %q: %w", draftKey, err)
+ }
+ content, err := io.ReadAll(getOut.Body)
+ getOut.Body.Close()
+ if err != nil {
+ return fmt.Errorf("read draft object %q: %w", draftKey, err)
}
- wg.Wait()
- if firstErr != nil {
- return firstErr
+ headOut, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{
+ Bucket: aws.String(sc.bucket),
+ Key: aws.String(draftKey),
+ })
+ if err != nil {
+ return fmt.Errorf("head draft object %q: %w", draftKey, err)
}
+ draftETag := aws.ToString(headOut.ETag)
- now := time.Now().UTC()
- fresh := make(map[string]*IndexV2Category)
- for _, r := range results {
- if r == nil {
- continue
+ geolocationParts := strings.Split(metadata.Geolocation, " ")
+ if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 {
+ return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string")
+ }
+ if len(geolocationParts) >= 2 {
+ if _, err := strconv.ParseFloat(geolocationParts[0], 64); err != nil {
+ return fmt.Errorf("invalid geolocation parameter, expected float for X: %w", err)
}
- if _, ok := fresh[r.catKey]; !ok {
- fresh[r.catKey] = &IndexV2Category{
- Pages: make(map[string]IndexEntry),
- }
+ if _, err := strconv.ParseFloat(geolocationParts[1], 64); err != nil {
+ return fmt.Errorf("invalid geolocation parameter, expected float for Y: %w", err)
+ }
+ }
+ if len(geolocationParts) == 3 {
+ if _, err := strconv.ParseFloat(geolocationParts[2], 64); err != nil {
+ return fmt.Errorf("invalid geolocation parameter, expected float for area error: %w", err)
}
- fresh[r.catKey].Pages[r.codename] = r.entry
- fresh[r.catKey].GeneratedAt = now
}
- merged, err := sc.loadAndMergeIndex(fresh)
- if err != nil {
- return err
+ medley := ""
+ if metadata.Medley != "" {
+ medley = fmt.Sprintf("%s %d", metadata.Medley, metadata.MedleyPart)
}
- idx := Index{
- SchemaVersion: IndexSchemaVersion,
- GeneratedAt: now,
- Categories: merged,
+ s3Metadata := map[string]string{
+ "title": metadata.Title,
+ "short-description": metadata.ShortDescription,
+ "action-date": metadata.ActionDate,
+ "published-time": metadata.PublishedTime.Format(time.RFC3339),
+ "thumbnail": metadata.Thumbnail,
+ "tags": strings.Join(metadata.Tags, ","),
+ "geolocation": metadata.Geolocation,
+ "medley": medley,
+ "metadata-last-update-etag": draftETag,
}
- body, err := json.Marshal(idx)
- if err != nil {
- return fmt.Errorf("marshal index: %w", err)
+ targetKey := draftKey
+ if sc.draftModeCfg.Enabled {
+ prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+ targetKey = sc.prefix + prodPath
}
_, err = sc.client.PutObject(context.Background(), &s3.PutObjectInput{
Bucket: aws.String(sc.bucket),
- Key: aws.String(IndexFileName),
- Body: bytes.NewReader(body),
- ContentType: aws.String("application/json; charset=utf-8"),
+ Key: aws.String(targetKey),
+ Body: bytes.NewReader(content),
+ ContentType: aws.String("text/markdown; charset=utf-8"),
+ Metadata: s3Metadata,
})
if err != nil {
- return fmt.Errorf("put index %q: %w", IndexFileName, err)
+ return fmt.Errorf("put S3 object %q: %w", targetKey, err)
}
- totalEntries := 0
- for _, c := range merged {
- totalEntries += len(c.Pages)
- }
- slog.Info("wrote index",
- slog.String("key", IndexFileName),
- slog.Int("categories", len(merged)),
- slog.Int("entries", totalEntries))
return nil
}
-func (sc *S3StorageClient) scanMedleys() ([]MedleyEntry, error) {
- out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{
+func (sc *S3StorageClient) CompareDraftAndProd(path string) bool {
+ draftKey := sc.prefix + path
+ prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+ prodKey := sc.prefix + prodPath
+
+ draftHead, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{
Bucket: aws.String(sc.bucket),
- Key: aws.String(MedleysIndexFileName),
+ Key: aws.String(draftKey),
})
if err != nil {
- return nil, fmt.Errorf("get %s: %w", MedleysIndexFileName, err)
- }
- defer out.Body.Close()
-
- raw, err := io.ReadAll(out.Body)
- if err != nil {
- return nil, fmt.Errorf("read %s: %w", MedleysIndexFileName, err)
+ return false
}
- var entries []MedleyEntry
- if err := json.Unmarshal(raw, &entries); err != nil {
- return nil, fmt.Errorf("unmarshal %s: %w", MedleysIndexFileName, err)
- }
-
- return entries, nil
-}
-
-func (sc *S3StorageClient) loadAndMergeIndex(fresh map[string]*IndexV2Category) (map[string]*IndexV2Category, error) {
- merged := make(map[string]*IndexV2Category)
-
- out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{
+ prodHead, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{
Bucket: aws.String(sc.bucket),
- Key: aws.String(IndexFileName),
+ Key: aws.String(prodKey),
})
- if err == nil {
- defer out.Body.Close()
- raw, readErr := io.ReadAll(out.Body)
- if readErr != nil {
- return nil, fmt.Errorf("read existing index: %w", readErr)
- }
-
- var legacyIdx Index
- if err := json.Unmarshal(raw, &legacyIdx); err != nil {
- return nil, fmt.Errorf("unmarshal existing index: %w", err)
- }
-
- switch legacyIdx.SchemaVersion {
- case 1:
- for k, v := range *legacyIdx.Categories.(*map[string]*IndexV1Category) {
- pages := make(map[string]IndexEntry, len(v.Pages))
- for _, page := range v.Pages {
- pages[page.Link[strings.LastIndex(page.Link, "/")+1:strings.LastIndex(page.Link, ".")]] = page
- }
- merged[k] = &IndexV2Category{
- GeneratedAt: v.GeneratedAt,
- Pages: pages,
- }
- }
- case 2:
- maps.Copy(merged, *legacyIdx.Categories.(*map[string]*IndexV2Category))
- default:
- slog.Warn("unknown index schema, discarding", slog.Int("schema_version", legacyIdx.SchemaVersion))
- }
- } else {
- var nsk *s3types.NoSuchKey
- if !errors.As(err, &nsk) {
- return nil, fmt.Errorf("get existing index: %w", err)
- }
+ if err != nil {
+ return true
}
- for k := range merged {
- if strings.HasPrefix(k, sc.prefix) {
- delete(merged, k)
- }
+ lastUpdateETag, ok := prodHead.Metadata["metadata-last-update-etag"]
+ if !ok {
+ return true
}
- maps.Copy(merged, fresh)
- return merged, nil
+ return aws.ToString(draftHead.ETag) != lastUpdateETag
}
diff --git a/internal/storage/storage_interface.go b/internal/storage/storage_interface.go
index 92a37e2..7134e30 100644
--- a/internal/storage/storage_interface.go
+++ b/internal/storage/storage_interface.go
@@ -1,17 +1,20 @@
package storage
import (
+ "io"
+
"github.com/SayaAndy/saya-today-article-metadata-add/config"
"github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
)
type StorageClient interface {
- Put(key string, content []byte, metadata *frontmatter.Metadata) error
- GetMetadata(key string) (metadata map[string]string, err error)
- BuildIndex() error
+ Scan() (paths []string, err error)
+ GetReader(path string) (reader io.ReadCloser, sz int64, err error)
+ WriteMetadata(path string, metadata *frontmatter.Metadata) error
+ CompareDraftAndProd(path string) (changed bool)
}
-var NewStorageClientMap = map[string]func(*config.StorageConfig) (StorageClient, error){
+var NewStorageClientMap = map[string]func(*config.StorageConfig, *config.DraftModeConfig) (StorageClient, error){
"b2": NewB2StorageClient,
"s3": NewS3StorageClient,
}
diff --git a/internal/transcoder/format_test.go b/internal/transcoder/format_test.go
deleted file mode 100644
index 9b25dc6..0000000
--- a/internal/transcoder/format_test.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package transcoder
-
-import (
- "os"
- "testing"
- "time"
-)
-
-func TestFormatContentMatchesCommitted(t *testing.T) {
- draftRaw, err := os.ReadFile("../../.draft/sevsk.md.draft")
- if err != nil {
- t.Fatal(err)
- }
- expected, err := os.ReadFile("../../.draft/sevsk.md")
- if err != nil {
- t.Fatal(err)
- }
-
- published, err := time.Parse(time.RFC3339, "2025-08-26T11:21:53+07:00")
- if err != nil {
- t.Fatal(err)
- }
-
- got := formatContent(draftRaw, published, true)
- if string(got) != string(expected) {
- t.Errorf("formatted output mismatch\n--- got ---\n%q\n--- want ---\n%q", got, expected)
- }
-}
diff --git a/internal/transcoder/markdown.go b/internal/transcoder/markdown.go
deleted file mode 100644
index 975c3bc..0000000
--- a/internal/transcoder/markdown.go
+++ /dev/null
@@ -1,164 +0,0 @@
-package transcoder
-
-import (
- "fmt"
- "strings"
-
- "github.com/yuin/goldmark"
- "github.com/yuin/goldmark/ast"
- "github.com/yuin/goldmark/extension"
- xast "github.com/yuin/goldmark/extension/ast"
- "github.com/yuin/goldmark/text"
-)
-
-// telegramMD parses CommonMark once and is reused across calls.
-var telegramMD = goldmark.New(goldmark.WithExtensions(extension.Strikethrough, extension.Linkify))
-
-// markdownToTelegramHTML converts CommonMark source into the limited HTML
-// subset Telegram accepts (parse_mode=HTML). Telegram supports only
-// b/i/u/s/a/code/pre/blockquote — block constructs with no Telegram tag
-// (paragraphs, headings, lists) are flattened to text + newlines. Anything
-// outside the subset is dropped rather than emitted, so a message is never
-// rejected for an unsupported tag.
-func markdownToTelegramHTML(src string) string {
- source := []byte(src)
- doc := telegramMD.Parser().Parse(text.NewReader(source))
-
- var b strings.Builder
- renderNodes(&b, doc, source)
-
- // Collapse the runs of blank lines block rendering can leave behind.
- out := strings.TrimSpace(b.String())
- for strings.Contains(out, "\n\n\n") {
- out = strings.ReplaceAll(out, "\n\n\n", "\n\n")
- }
- return out
-}
-
-func renderNodes(b *strings.Builder, parent ast.Node, source []byte) {
- for n := parent.FirstChild(); n != nil; n = n.NextSibling() {
- renderNode(b, n, source)
- }
-}
-
-func renderNode(b *strings.Builder, n ast.Node, source []byte) {
- switch node := n.(type) {
- case *ast.Document:
- renderNodes(b, node, source)
-
- case *ast.Paragraph, *ast.TextBlock:
- renderNodes(b, node, source)
- b.WriteString("\n\n")
-
- case *ast.Heading:
- // Telegram has no headings; render the line in bold.
- b.WriteString("<b>")
- renderNodes(b, node, source)
- b.WriteString("</b>\n\n")
-
- case *ast.Blockquote:
- b.WriteString("<blockquote>")
- renderNodes(b, node, source)
- trimTrailingNewlines(b)
- b.WriteString("</blockquote>\n\n")
-
- case *ast.List:
- renderList(b, node, source)
- b.WriteString("\n")
-
- case *ast.FencedCodeBlock, *ast.CodeBlock:
- b.WriteString("<pre>")
- writeRawLines(b, n, source)
- b.WriteString("</pre>\n\n")
-
- case *ast.ThematicBreak:
- // horizontal rule — nothing meaningful in a Telegram message
-
- // --- inline ---
- case *ast.Text:
- b.WriteString(escapeHTML(string(node.Segment.Value(source))))
- if node.HardLineBreak() || node.SoftLineBreak() {
- b.WriteByte('\n')
- }
- case *ast.String:
- b.WriteString(escapeHTML(string(node.Value)))
-
- case *ast.Emphasis:
- tag := "i"
- if node.Level == 2 {
- tag = "b"
- }
- fmt.Fprintf(b, "<%s>", tag)
- renderNodes(b, node, source)
- fmt.Fprintf(b, "</%s>", tag)
-
- case *xast.Strikethrough:
- b.WriteString("<s>")
- renderNodes(b, node, source)
- b.WriteString("</s>")
-
- case *ast.CodeSpan:
- b.WriteString("<code>")
- renderNodes(b, node, source)
- b.WriteString("</code>")
-
- case *ast.Link:
- fmt.Fprintf(b, `<a href="%s">`, escapeHTML(string(node.Destination)))
- renderNodes(b, node, source)
- b.WriteString("</a>")
-
- case *ast.AutoLink:
- url := string(node.URL(source))
- fmt.Fprintf(b, `<a href="%s">%s</a>`, escapeHTML(url), escapeHTML(url))
-
- case *ast.Image:
- // Images can't render inline in text; keep the alt text only.
- renderNodes(b, node, source)
-
- case *ast.RawHTML, *ast.HTMLBlock:
- // Drop raw HTML — it is almost certainly not in Telegram's tag subset.
-
- default:
- // Unknown node: recurse so inline text inside it is not lost.
- renderNodes(b, n, source)
- }
-}
-
-func renderList(b *strings.Builder, list *ast.List, source []byte) {
- i := list.Start
- for item := list.FirstChild(); item != nil; item = item.NextSibling() {
- if list.IsOrdered() {
- fmt.Fprintf(b, "%d. ", i)
- i++
- } else {
- b.WriteString("• ")
- }
- renderNodes(b, item, source)
- trimTrailingNewlines(b)
- b.WriteByte('\n')
- }
-}
-
-func writeRawLines(b *strings.Builder, n ast.Node, source []byte) {
- lines := n.Lines()
- for i := 0; i < lines.Len(); i++ {
- seg := lines.At(i)
- b.WriteString(escapeHTML(string(seg.Value(source))))
- }
-}
-
-func trimTrailingNewlines(b *strings.Builder) {
- s := strings.TrimRight(b.String(), "\n")
- b.Reset()
- b.WriteString(s)
-}
-
-// escapeHTML escapes the three characters Telegram's HTML parser treats as
-// markup. Quotes are escaped too so the value is safe inside an href="...".
-func escapeHTML(s string) string {
- s = strings.ReplaceAll(s, "&", "&amp;")
- s = strings.ReplaceAll(s, "<", "&lt;")
- s = strings.ReplaceAll(s, ">", "&gt;")
- s = strings.ReplaceAll(s, `"`, "&quot;")
- return s
-}
diff --git a/internal/transcoder/sayauz.go b/internal/transcoder/sayauz.go
deleted file mode 100644
index ee888e5..0000000
--- a/internal/transcoder/sayauz.go
+++ /dev/null
@@ -1,138 +0,0 @@
-package transcoder
-
-import (
- "fmt"
- "log/slog"
- "os"
- "path/filepath"
- "strings"
- "time"
-
- "github.com/SayaAndy/saya-today-article-metadata-add/config"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage"
-)
-
-type SayauzTranscoder struct {
- category string
- storage storage.StorageClient
-}
-
-func NewSayauzTranscoder(cfg any) (Transcoder, error) {
- sayauzCfg, ok := cfg.(*config.SayauzConfig)
- if !ok {
- return nil, fmt.Errorf("invalid config for sayauz transcoder")
- }
-
- newClient, ok := storage.NewStorageClientMap[sayauzCfg.Storage.Type]
- if !ok {
- return nil, fmt.Errorf("unsupported storage type %q for sayauz transcoder", sayauzCfg.Storage.Type)
- }
- storageClient, err := newClient(&sayauzCfg.Storage)
- if err != nil {
- return nil, fmt.Errorf("init storage client: %w", err)
- }
-
- return &SayauzTranscoder{
- category: sayauzCfg.Category,
- storage: storageClient,
- }, nil
-}
-
-func (t *SayauzTranscoder) Name() string { return "sayauz" }
-
-func (t *SayauzTranscoder) Transcode(doc *draft.Document) error {
- if doc.Metadata == nil {
- return fmt.Errorf("draft %q has no frontmatter metadata", doc.SourcePath)
- }
-
- prodKey := t.category + "/" + doc.Codename + ".md"
-
- // publishedTime is the first-publish time: reuse an existing value (from the
- // draft frontmatter, else from the already-published object), otherwise stamp now.
- inject := doc.Metadata.PublishedTime.IsZero()
- if inject {
- published := time.Time{}
- if meta, err := t.storage.GetMetadata(prodKey); err == nil {
- if v := meta["published-time"]; v != "" {
- if parsed, err := time.Parse(time.RFC3339, v); err == nil {
- published = parsed
- }
- }
- } else {
- slog.Warn("could not read existing metadata for publishedTime, stamping now",
- slog.String("key", prodKey), slog.String("error", err.Error()))
- }
- if published.IsZero() {
- published = time.Now()
- }
- doc.Metadata.PublishedTime = published
- }
-
- formatted := formatContent(doc.RawContent, doc.Metadata.PublishedTime, inject)
-
- localPath := filepath.Join(filepath.Dir(doc.SourcePath), doc.Codename+".md")
- if err := os.WriteFile(localPath, formatted, 0o644); err != nil {
- return fmt.Errorf("write formatted file %q: %w", localPath, err)
- }
-
- if err := t.storage.Put(prodKey, formatted, doc.Metadata); err != nil {
- return fmt.Errorf("upload %q: %w", prodKey, err)
- }
-
- slog.Info("sayauz published page",
- slog.String("key", prodKey),
- slog.String("local", localPath))
- return nil
-}
-
-func (t *SayauzTranscoder) Finalize() error {
- return t.storage.BuildIndex()
-}
-
-// formatContent renders a draft into its published form: standalone "..."
-// separator lines become blank lines (trailing ones are dropped), and when
-// inject is true a "publishedTime:" line is added after "actionDate:" in the
-// frontmatter. Frontmatter, galleries and "---" rules are otherwise untouched.
-func formatContent(raw []byte, published time.Time, inject bool) []byte {
- lines := strings.Split(string(raw), "\n")
- out := make([]string, 0, len(lines)+1)
-
- inFrontmatter := false
- frontmatterDone := false
-
- for i, line := range lines {
- trimmed := strings.TrimSpace(line)
-
- if !frontmatterDone {
- if i == 0 && trimmed == "---" {
- inFrontmatter = true
- out = append(out, line)
- continue
- }
- if inFrontmatter {
- out = append(out, line)
- switch {
- case trimmed == "---":
- inFrontmatter = false
- frontmatterDone = true
- case inject && strings.HasPrefix(trimmed, "actionDate:"):
- out = append(out, "publishedTime: "+published.Format(time.RFC3339))
- }
- continue
- }
- }
-
- if trimmed == "..." {
- out = append(out, "")
- continue
- }
- out = append(out, line)
- }
-
- for len(out) > 0 && out[len(out)-1] == "" {
- out = out[:len(out)-1]
- }
-
- return []byte(strings.Join(out, "\n"))
-}
diff --git a/internal/transcoder/telegram.go b/internal/transcoder/telegram.go
deleted file mode 100644
index b521fb0..0000000
--- a/internal/transcoder/telegram.go
+++ /dev/null
@@ -1,584 +0,0 @@
-package transcoder
-
-import (
- "bytes"
- "encoding/json"
- "fmt"
- "io"
- "log/slog"
- "mime/multipart"
- "net/http"
- "os"
- "path/filepath"
- "slices"
- "strconv"
- "strings"
- "time"
-
- "github.com/SayaAndy/saya-today-article-metadata-add/config"
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft"
-)
-
-const (
- telegramMediaGroupLimit = 10
- // telegramCaptionLimit is the max length (runes) of a media caption.
- telegramCaptionLimit = 1024
- // telegramMaxAttempts caps retries when the API returns 429.
- telegramMaxAttempts = 5
-)
-
-type TelegramTranscoder struct {
- botToken string
- channelID int
- groupID int
- lastUpdateID int
- imageBaseURL string
- previewBaseURL string
- previewExtension string
- stateDir string
- http *http.Client
-}
-
-func NewTelegramTranscoder(cfg any) (Transcoder, error) {
- tgCfg, ok := cfg.(*config.TelegramConfig)
- if !ok {
- return nil, fmt.Errorf("invalid config for telegram transcoder")
- }
-
- transcoder := &TelegramTranscoder{
- botToken: tgCfg.BotToken,
- channelID: tgCfg.ChannelID,
- groupID: tgCfg.GroupID,
- lastUpdateID: -1,
- imageBaseURL: strings.TrimRight(tgCfg.ImageBaseURL, "/"),
- previewBaseURL: strings.TrimRight(tgCfg.PreviewBaseURL, "/"),
- previewExtension: tgCfg.PreviewExtension,
- stateDir: tgCfg.StateDir,
- http: &http.Client{Timeout: time.Minute},
- }
-
- if _, err := transcoder.getUpdates(); err != nil {
- return nil, fmt.Errorf("failed to get last update id for checkpoint: %w", err)
- }
-
- return transcoder, nil
-}
-
-func (t *TelegramTranscoder) Name() string { return "telegram" }
-
-func (t *TelegramTranscoder) Finalize() error { return nil }
-
-// telegramState records, per draft, which section indices have already been
-// posted and the message ids they produced. Posting is append-only: a section
-// already present here is never re-sent.
-type telegramState struct {
- ChannelID string `json:"channelID"`
- Posted map[string][]int `json:"posted"`
-}
-
-func (t *TelegramTranscoder) Transcode(doc *draft.Document) error {
- statePath := filepath.Join(t.stateDir, doc.Codename+".telegram-state.json")
- state, err := loadTelegramState(statePath)
- if err != nil {
- return fmt.Errorf("load telegram state %q: %w", statePath, err)
- }
-
- if state.ChannelID != "" && state.ChannelID != strconv.Itoa(t.channelID) {
- slog.Warn("telegram channel changed, re-posting all sections to the new channel",
- slog.String("codename", doc.Codename),
- slog.String("old", state.ChannelID),
- slog.Int("new", t.channelID))
- state.Posted = map[string][]int{}
- }
- state.ChannelID = strconv.Itoa(t.channelID)
-
- for _, section := range doc.Sections {
- idx := strconv.Itoa(section.Index)
- if _, done := state.Posted[idx]; done {
- continue
- }
-
- var msgIDs []int
- pendingText := ""
-
- // flushText sends any buffered text as its own message.
- flushText := func() error {
- if pendingText == "" {
- return nil
- }
- id, err := t.sendMessage(pendingText)
- if err != nil {
- return err
- }
- msgIDs = append(msgIDs, id)
- pendingText = ""
- return nil
- }
-
- for _, block := range section.Blocks {
- switch block.Kind {
- case draft.TextBlock:
- text := stripRules(block.Text)
- if text == "" {
- continue
- }
- // flush any earlier text before buffering this one
- if err := flushText(); err != nil {
- return fmt.Errorf("send section %d text: %w", section.Index, err)
- }
- pendingText = text
- case draft.GalleryBlock:
- // Attach the immediately preceding text as the album caption when
- // it fits; otherwise send it as its own message first.
- caption := ""
- if pendingText != "" {
- if len([]rune(pendingText)) <= telegramCaptionLimit {
- caption = pendingText
- pendingText = ""
- } else if err := flushText(); err != nil {
- return fmt.Errorf("send section %d text: %w", section.Index, err)
- }
- }
- for i, group := range splitPhotos(block.Gallery.Photos, telegramMediaGroupLimit) {
- groupCaption := ""
- if i == 0 {
- groupCaption = caption
- }
- ids, err := t.sendMediaGroup(group, groupCaption, "photo", t.channelID, 0)
- if err != nil {
- return fmt.Errorf("send section %d gallery: %w", section.Index, err)
- }
- msgIDs = append(msgIDs, ids...)
- if err := flushText(); err != nil {
- return fmt.Errorf("send section %d text: %w", section.Index, err)
- }
- if len(msgIDs) > 0 {
- time.Sleep(5 * time.Second)
- forwardedID, err := t.getForwardedID(msgIDs)
- if err != nil {
- return fmt.Errorf("send full documents for section %d gallery: %w", section.Index, err)
- }
- if forwardedID == 0 {
- return fmt.Errorf("fail to get forwarded id of section %d gallery", section.Index)
- }
- _, err = t.sendMediaGroup(group, "", "document", t.groupID, forwardedID)
- if err != nil {
- return fmt.Errorf("send section %d documents: %w", section.Index, err)
- }
- }
- }
- }
- }
-
- state.Posted[idx] = msgIDs
- if err := saveTelegramState(statePath, state); err != nil {
- return fmt.Errorf("save telegram state %q: %w", statePath, err)
- }
- slog.Info("telegram posted section",
- slog.String("codename", doc.Codename),
- slog.Int("section", section.Index),
- slog.Int("messages", len(msgIDs)))
- }
-
- return nil
-}
-
-func loadTelegramState(path string) (*telegramState, error) {
- raw, err := os.ReadFile(path)
- if err != nil {
- if os.IsNotExist(err) {
- return &telegramState{Posted: map[string][]int{}}, nil
- }
- return nil, err
- }
- var state telegramState
- if err := json.Unmarshal(raw, &state); err != nil {
- return nil, err
- }
- if state.Posted == nil {
- state.Posted = map[string][]int{}
- }
- return &state, nil
-}
-
-func saveTelegramState(path string, state *telegramState) error {
- if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
- return err
- }
- raw, err := json.MarshalIndent(state, "", " ")
- if err != nil {
- return err
- }
- return os.WriteFile(path, raw, 0o644)
-}
-
-// stripRules drops standalone "---" horizontal-rule lines (meaningless in a
-// Telegram message) and trims the remaining text.
-func stripRules(text string) string {
- lines := strings.Split(text, "\n")
- kept := lines[:0]
- for _, line := range lines {
- if strings.TrimSpace(line) == "---" {
- continue
- }
- kept = append(kept, line)
- }
- return strings.TrimSpace(strings.Join(kept, "\n"))
-}
-
-// splitPhotos divides photos into near-equal groups each no larger than limit.
-func splitPhotos(photos []draft.Photo, limit int) [][]draft.Photo {
- n := len(photos)
- if n == 0 {
- return nil
- }
- groupCount := (n + limit - 1) / limit
- base := n / groupCount
- rem := n % groupCount
-
- var groups [][]draft.Photo
- start := 0
- for g := range groupCount {
- size := base
- if g < rem {
- size++
- }
- groups = append(groups, photos[start:start+size])
- start += size
- }
- return groups
-}
-
-type inputMediaPhoto struct {
- Type string `json:"type"`
- Media string `json:"media"`
- Caption string `json:"caption,omitempty"`
- ParseMode string `json:"parse_mode,omitempty"`
-}
-
-func (t *TelegramTranscoder) photoURL(key string) string {
- return t.imageBaseURL + "/" + key
-}
-
-func (t *TelegramTranscoder) previewURL(key string) string {
- base := t.previewBaseURL
- if base == "" {
- base = t.imageBaseURL
- }
- if t.previewExtension != "" {
- key = strings.TrimSuffix(key, filepath.Ext(key)) + t.previewExtension
- }
- return base + "/" + key
-}
-
-func (t *TelegramTranscoder) sendMessage(text string) (int, error) {
- payload := map[string]any{
- "chat_id": strconv.Itoa(t.channelID),
- "text": markdownToTelegramHTML(text),
- "parse_mode": "HTML",
- "disable_web_page_preview": true,
- }
- var result struct {
- MessageID int `json:"message_id"`
- }
- if err := t.call("sendMessage", payload, &result); err != nil {
- return 0, err
- }
- return result.MessageID, nil
-}
-
-func (t *TelegramTranscoder) sendMediaGroup(photos []draft.Photo, caption string, sendAsType string, id int, replyTo int) ([]int, error) {
- media := make([]inputMediaPhoto, 0, len(photos))
- files := map[string]uploadFile{}
- for i, p := range photos {
- item := inputMediaPhoto{Type: sendAsType}
- if sendAsType == "document" {
- // Upload the original file directly rather than passing Telegram a
- // URL to fetch: full-resolution originals routinely fail Telegram's
- // server-side downloader with WEBPAGE_CURL_FAILED.
- data, err := t.downloadPhoto(p.Key)
- if err != nil {
- return nil, fmt.Errorf("download %q for upload: %w", p.Key, err)
- }
- field := fmt.Sprintf("file%d", i)
- files[field] = uploadFile{name: filepath.Base(p.Key), data: data}
- item.Media = "attach://" + field
- } else {
- item.Media = t.previewURL(p.Key)
- }
- // Telegram shows an album's caption in the feed only when exactly one
- // item is captioned. So the section text goes on the first photo and
- // every other photo is left uncaptioned (per-photo captions dropped).
- if i == 0 && caption != "" {
- item.Caption = markdownToTelegramHTML(caption)
- item.ParseMode = "HTML"
- }
- media = append(media, item)
- }
-
- var result []struct {
- MessageID int `json:"message_id"`
- }
-
- if len(files) > 0 {
- mediaJSON, err := json.Marshal(media)
- if err != nil {
- return nil, err
- }
- fields := map[string]string{
- "chat_id": strconv.Itoa(id),
- "media": string(mediaJSON),
- }
- if replyTo != 0 {
- replyJSON, err := json.Marshal(map[string]any{"message_id": replyTo})
- if err != nil {
- return nil, err
- }
- fields["reply_parameters"] = string(replyJSON)
- }
- if err := t.callMultipart("sendMediaGroup", fields, files, &result); err != nil {
- return nil, err
- }
- } else {
- payload := map[string]any{
- "chat_id": strconv.Itoa(id),
- "media": media,
- }
- if replyTo != 0 {
- payload["reply_parameters"] = map[string]any{
- "message_id": replyTo,
- }
- }
- if err := t.call("sendMediaGroup", payload, &result); err != nil {
- return nil, err
- }
- }
-
- ids := make([]int, 0, len(result))
- for _, r := range result {
- ids = append(ids, r.MessageID)
- }
- return ids, nil
-}
-
-type uploadFile struct {
- name string
- data []byte
-}
-
-// downloadPhoto fetches the original bytes for a photo key so they can be
-// uploaded directly to Telegram instead of fetched server-side by Telegram.
-func (t *TelegramTranscoder) downloadPhoto(key string) ([]byte, error) {
- url := t.photoURL(key)
- resp, err := t.http.Get(url)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("unexpected status %d fetching %q", resp.StatusCode, url)
- }
- return io.ReadAll(resp.Body)
-}
-
-func (t *TelegramTranscoder) getForwardedID(channelMessageIDs []int) (int, error) {
- var payloads []getUpdatesPayload
- var err error
- slog.Debug("search for forwarded message id", slog.Any("original_id", channelMessageIDs), slog.Int("group_id", t.groupID), slog.Int("channel_id", t.channelID))
-
- for range 3 {
- for payloads, err = t.getUpdates(); len(payloads) > 0 && err == nil; payloads, err = t.getUpdates() {
- for _, payload := range payloads {
- if payload.Message.MessageOrigin.Type == "channel" &&
- payload.Message.MessageOrigin.Chat.ID == t.channelID &&
- slices.Contains(channelMessageIDs, payload.Message.MessageOrigin.MessageID) &&
- payload.Message.Chat.ID == t.groupID {
- return payload.Message.MessageID, nil
- }
- }
- }
- time.Sleep(3 * time.Second)
- }
-
- return 0, err
-}
-
-type getUpdatesPayload struct {
- UpdateID int `json:"update_id"`
- Message struct {
- MessageID int `json:"message_id"`
- Chat struct {
- ID int `json:"id"`
- Title string `json:"title"`
- Type string `json:"type"`
- } `json:"chat"`
- MessageOrigin struct {
- Type string `json:"type"`
- Date int `json:"date"`
- Chat struct {
- ID int `json:"id"`
- Title string `json:"title"`
- Type string `json:"type"`
- } `json:"chat"`
- MessageID int `json:"message_id"`
- } `json:"forward_origin"`
- Date int `json:"date"`
- Text string `json:"text"`
- } `json:"message"`
-}
-
-func (t *TelegramTranscoder) getUpdates() ([]getUpdatesPayload, error) {
- var out []getUpdatesPayload
-
- payload := struct {
- Offset int `json:"offset"`
- }{
- Offset: t.lastUpdateID,
- }
-
- if err := t.call("getUpdates", payload, &out); err != nil {
- return nil, fmt.Errorf("failed to transcode received message: %w", err)
- }
-
- for _, p := range out {
- pjson, _ := json.Marshal(p)
- slog.Debug("parsed new update", slog.String("payload", string(pjson)))
- if p.UpdateID > t.lastUpdateID {
- t.lastUpdateID = p.UpdateID
- }
- }
- t.lastUpdateID++
-
- return out, nil
-}
-
-// handleResponse decodes a Telegram API response envelope. When the API asks us
-// to back off (HTTP 429) it returns a positive wait duration and a nil error so
-// the caller can retry; otherwise it unmarshals the result into out (if any) or
-// returns the API error verbatim.
-func (t *TelegramTranscoder) handleResponse(method string, resp *http.Response, attempt int, out any) (time.Duration, error) {
- var envelope struct {
- OK bool `json:"ok"`
- ErrorCode int `json:"error_code"`
- Description string `json:"description"`
- Parameters struct {
- RetryAfter int `json:"retry_after"`
- } `json:"parameters"`
- Result json.RawMessage `json:"result"`
- }
- decodeErr := json.NewDecoder(resp.Body).Decode(&envelope)
- resp.Body.Close()
- if decodeErr != nil {
- return 0, fmt.Errorf("decode telegram response (%s): %w", method, decodeErr)
- }
-
- envelopeBytes, _ := json.Marshal(envelope)
- slog.Debug("telegram response", slog.String("method", method), slog.String("response", string(envelopeBytes)))
-
- if envelope.OK {
- if out == nil {
- return 0, nil
- }
- return 0, json.Unmarshal(envelope.Result, out)
- }
-
- // 429 Too Many Requests: Telegram tells us how long to wait in
- // parameters.retry_after. Back off and retry instead of failing.
- if envelope.ErrorCode == 429 && attempt < telegramMaxAttempts {
- wait := time.Duration(envelope.Parameters.RetryAfter) * time.Second
- if wait <= 0 {
- wait = time.Second
- }
- slog.Warn("telegram rate limited, backing off",
- slog.String("method", method),
- slog.Int("attempt", attempt),
- slog.Duration("retry_after", wait))
- return wait, nil
- }
-
- return 0, fmt.Errorf("telegram %s failed: %s", method, envelope.Description)
-}
-
-// call invokes a Telegram Bot API method with a JSON body and decodes the
-// "result" field into out. The API's error description is surfaced verbatim.
-func (t *TelegramTranscoder) call(method string, payload any, out any) error {
- body, err := json.Marshal(payload)
- if err != nil {
- return err
- }
-
- url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", t.botToken, method)
-
- for attempt := 1; ; attempt++ {
- req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(body))
- if err != nil {
- return err
- }
- req.Header.Set("Content-Type", "application/json")
-
- slog.Debug("send message", slog.String("method", method), slog.String("request", string(body)))
-
- resp, err := t.http.Do(req)
- if err != nil {
- return err
- }
-
- wait, err := t.handleResponse(method, resp, attempt, out)
- if wait > 0 {
- time.Sleep(wait)
- continue
- }
- return err
- }
-}
-
-// callMultipart invokes a Telegram Bot API method using multipart/form-data,
-// uploading the given files directly instead of handing Telegram URLs to fetch
-// server-side. fields carries the non-file form values (chat_id, media JSON,
-// reply_parameters, ...); files maps a form field name to its bytes and upload
-// filename, referenced from the media JSON via attach://<field>.
-func (t *TelegramTranscoder) callMultipart(method string, fields map[string]string, files map[string]uploadFile, out any) error {
- url := fmt.Sprintf("https://api.telegram.org/bot%s/%s", t.botToken, method)
-
- for attempt := 1; ; attempt++ {
- var body bytes.Buffer
- w := multipart.NewWriter(&body)
- for k, v := range fields {
- if err := w.WriteField(k, v); err != nil {
- return err
- }
- }
- for field, f := range files {
- part, err := w.CreateFormFile(field, f.name)
- if err != nil {
- return err
- }
- if _, err := part.Write(f.data); err != nil {
- return err
- }
- }
- if err := w.Close(); err != nil {
- return err
- }
-
- req, err := http.NewRequest(http.MethodPost, url, &body)
- if err != nil {
- return err
- }
- req.Header.Set("Content-Type", w.FormDataContentType())
-
- slog.Debug("send media group upload", slog.String("method", method), slog.Int("files", len(files)))
-
- resp, err := t.http.Do(req)
- if err != nil {
- return err
- }
-
- wait, err := t.handleResponse(method, resp, attempt, out)
- if wait > 0 {
- time.Sleep(wait)
- continue
- }
- return err
- }
-}
diff --git a/internal/transcoder/transcoder.go b/internal/transcoder/transcoder.go
deleted file mode 100644
index 3dbdf7e..0000000
--- a/internal/transcoder/transcoder.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package transcoder
-
-import (
- "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft"
-)
-
-// Transcoder turns a parsed draft into a published artifact (an S3 page, a
-// Telegram thread, ...). Transcode runs once per draft; Finalize runs once
-// after all drafts have been processed.
-type Transcoder interface {
- Name() string
- Transcode(doc *draft.Document) error
- Finalize() error
-}
-
-var NewTranscoderMap = map[string]func(cfg any) (Transcoder, error){
- "sayauz": NewSayauzTranscoder,
- "telegram": NewTelegramTranscoder,
-}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-17 09:13:48 +0000