From 684d0b6a57d2eb79730ade63baccdc6e59bebc29 Mon Sep 17 00:00:00 2001 From: Saya Andy Date: Sat, 1 Aug 2026 19:58:57 +0700 Subject: feat: repurpose metadata parser as article transcoder for saya.uz and telegram --- main.go | 105 ++++++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 45 deletions(-) (limited to 'main.go') diff --git a/main.go b/main.go index 04b0a4d..6648aeb 100644 --- a/main.go +++ b/main.go @@ -2,13 +2,16 @@ package main import ( "flag" + "io/fs" "log/slog" "os" + "path/filepath" + "strings" "sync" "github.com/SayaAndy/saya-today-article-metadata-add/config" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/transcoder" ) var configPath = flag.String("c", "config.json", "Path to the configuration file") @@ -16,8 +19,8 @@ var configPath = flag.String("c", "config.json", "Path to the configuration file func main() { flag.Parse() - cfg := &config.Config{} - if err := config.LoadConfig(*configPath, cfg); err != nil { + cfg, err := config.InitConfig(*configPath) + if err != nil { slog.Error("fail to load configuration", slog.String("error", err.Error())) os.Exit(1) } @@ -25,73 +28,85 @@ func main() { slog.SetLogLoggerLevel(cfg.LogLevel) slog.Info("starting metadata extractor...") - storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage, &cfg.DraftMode) - if err != nil { - slog.Error("fail to initialize input client", slog.String("error", err.Error())) - os.Exit(1) + transcoders := make([]transcoder.Transcoder, 0, len(cfg.Transcoders)) + for _, tcCfg := range cfg.Transcoders { + newTranscoder, ok := transcoder.NewTranscoderMap[tcCfg.Type] + if !ok { + slog.Error("unsupported transcoder type", slog.String("type", tcCfg.Type)) + os.Exit(1) + } + t, err := newTranscoder(tcCfg.Config) + if err != nil { + slog.Error("fail to initialize transcoder", slog.String("type", tcCfg.Type), slog.String("error", err.Error())) + os.Exit(1) + } + transcoders = append(transcoders, t) + slog.Info("initialized transcoder", slog.String("type", tcCfg.Type)) } - generalLogger := slog.With( - slog.String("storage_type", cfg.Storage.Type), - ) - generalLogger.Info("initialized storage client") - - files, err := storageClient.Scan() + var drafts []string + err = filepath.WalkDir(cfg.DraftDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(path, cfg.DraftSuffix) { + drafts = append(drafts, path) + } + return nil + }) if err != nil { - generalLogger.Error("fail to scan input files", slog.String("error", err.Error())) + slog.Error("fail to scan draft directory", slog.String("dir", cfg.DraftDir), slog.String("error", err.Error())) os.Exit(1) } - generalLogger.Info("scanned files", slog.Int("file_count", len(files))) + slog.Info("scanned drafts", slog.Int("draft_count", len(drafts))) semaphore := make(chan struct{}, cfg.MaxConcurrentJobs) var wg sync.WaitGroup - wg.Add(len(files)) + wg.Add(len(drafts)) - for i, file := range files { + for _, draftPath := range drafts { semaphore <- struct{}{} - go func(index int, inputName string) { + go func(path string) { defer wg.Done() defer func() { <-semaphore }() - if cfg.DraftMode.Enabled && !storageClient.CompareDraftAndProd(inputName) { - generalLogger.Debug("skip a draft because prod object is identical to it", slog.String("file", inputName)) - return - } - generalLogger.Debug("processing a file", slog.String("file", inputName)) - reader, sz, err := storageClient.GetReader(inputName) - if err != nil { - generalLogger.Warn("fail to get reader for a file", slog.String("file", inputName), slog.String("error", err.Error())) - return - } - defer reader.Close() + codename := strings.TrimSuffix(filepath.Base(path), cfg.DraftSuffix) + fileLogger := slog.With(slog.String("draft", path), slog.String("codename", codename)) - content := make([]byte, sz) - ln, err := reader.Read(content) + content, err := os.ReadFile(path) if err != nil { - generalLogger.Warn("fail to read content from a file", slog.String("file", inputName), slog.String("error", err.Error())) + fileLogger.Warn("fail to read draft", slog.String("error", err.Error())) return } - generalLogger.Debug("read content from a file", - slog.String("file", inputName), - slog.Int64("expected_size", sz), - slog.Int("output_size", ln)) - metadata, _, err := frontmatter.ParseFrontmatter(content) + doc, err := draft.ParseDraft(path, codename, content) if err != nil { - generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", inputName), slog.String("error", err.Error())) + fileLogger.Warn("fail to parse draft", slog.String("error", err.Error())) return } - - if metadata == nil { - generalLogger.Info("skip a file due to it not having metadata", slog.String("file", inputName)) + if doc.Metadata == nil { + fileLogger.Info("skip draft without frontmatter metadata") return } - if err = storageClient.WriteMetadata(inputName, metadata); err != nil { - generalLogger.Warn("fail to write metadata to a file", slog.String("file", inputName), slog.String("error", err.Error())) + for _, t := range transcoders { + if err := t.Transcode(doc); err != nil { + fileLogger.Warn("transcoder failed", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) + } } - }(i, file) + }(draftPath) } wg.Wait() + + for _, t := range transcoders { + if err := t.Finalize(); err != nil { + slog.Error("fail to finalize transcoder", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) + os.Exit(1) + } + slog.Info("finalized transcoder", slog.String("transcoder", t.Name())) + } } -- cgit v1.3.1+13