Diffstat (limited to 'main.go')
| -rw-r--r-- | main.go | 125 |
1 files changed, 76 insertions, 49 deletions
@@ -2,12 +2,16 @@ package main import ( "flag" + "io/fs" "log/slog" "os" + "path/filepath" + "strings" + "sync" "github.com/SayaAndy/saya-today-article-metadata-add/config" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/transcoder" ) var configPath = flag.String("c", "config.json", "Path to the configuration file") @@ -15,8 +19,8 @@ var configPath = flag.String("c", "config.json", "Path to the configuration file func main() { flag.Parse() - cfg := &config.Config{} - if err := config.LoadConfig(*configPath, cfg); err != nil { + cfg, err := config.InitConfig(*configPath) + if err != nil { slog.Error("fail to load configuration", slog.String("error", err.Error())) os.Exit(1) } @@ -24,62 +28,85 @@ func main() { slog.SetLogLoggerLevel(cfg.LogLevel) slog.Info("starting metadata extractor...") - storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage) - if err != nil { - slog.Error("fail to initialize input client", slog.String("error", err.Error())) - os.Exit(1) + transcoders := make([]transcoder.Transcoder, 0, len(cfg.Transcoders)) + for _, tcCfg := range cfg.Transcoders { + newTranscoder, ok := transcoder.NewTranscoderMap[tcCfg.Type] + if !ok { + slog.Error("unsupported transcoder type", slog.String("type", tcCfg.Type)) + os.Exit(1) + } + t, err := newTranscoder(tcCfg.Config) + if err != nil { + slog.Error("fail to initialize transcoder", slog.String("type", tcCfg.Type), slog.String("error", err.Error())) + os.Exit(1) + } + transcoders = append(transcoders, t) + slog.Info("initialized transcoder", slog.String("type", tcCfg.Type)) } - generalLogger := slog.With( - slog.String("storage_type", cfg.Storage.Type), - ) - generalLogger.Info("initialized storage client") - - files, err := storageClient.Scan() + var drafts []string + err = filepath.WalkDir(cfg.DraftDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + if strings.HasSuffix(path, cfg.DraftSuffix) { + drafts = append(drafts, path) + } + return nil + }) if err != nil { - generalLogger.Error("fail to scan input files", slog.String("error", err.Error())) + slog.Error("fail to scan draft directory", slog.String("dir", cfg.DraftDir), slog.String("error", err.Error())) os.Exit(1) } - generalLogger.Info("scanned files", slog.Int("file_count", len(files))) + slog.Info("scanned drafts", slog.Int("draft_count", len(drafts))) - for _, file := range files { - if !storageClient.FileHasChanged(file) { - generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file)) - continue - } - generalLogger.Debug("processing a file", slog.String("file", file)) + semaphore := make(chan struct{}, cfg.MaxConcurrentJobs) + var wg sync.WaitGroup + wg.Add(len(drafts)) - reader, sz, err := storageClient.GetReader(file) - if err != nil { - generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error())) - continue - } - defer reader.Close() + for _, draftPath := range drafts { + semaphore <- struct{}{} + go func(path string) { + defer wg.Done() + defer func() { <-semaphore }() - content := make([]byte, sz) - ln, err := reader.Read(content) - if err != nil { - generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error())) - continue - } - generalLogger.Debug("read content from a file", - slog.String("file", file), - slog.Int64("expected_size", sz), - slog.Int("output_size", ln)) + codename := strings.TrimSuffix(filepath.Base(path), cfg.DraftSuffix) + fileLogger := slog.With(slog.String("draft", path), slog.String("codename", codename)) - metadata, _, err := frontmatter.ParseFrontmatter(content) - if err != nil { - generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error())) - continue - } + content, err := os.ReadFile(path) + if err != nil { + fileLogger.Warn("fail to read draft", slog.String("error", err.Error())) + return + } - if metadata == nil { - generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file)) - continue - } + doc, err := draft.ParseDraft(path, codename, content) + if err != nil { + fileLogger.Warn("fail to parse draft", slog.String("error", err.Error())) + return + } + if doc.Metadata == nil { + fileLogger.Info("skip draft without frontmatter metadata") + return + } + + for _, t := range transcoders { + if err := t.Transcode(doc); err != nil { + fileLogger.Warn("transcoder failed", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) + } + } + }(draftPath) + } + + wg.Wait() - if err = storageClient.WriteMetadata(file, metadata); err != nil { - generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error())) + for _, t := range transcoders { + if err := t.Finalize(); err != nil { + slog.Error("fail to finalize transcoder", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) + os.Exit(1) } + slog.Info("finalized transcoder", slog.String("transcoder", t.Name())) } } |