Diffstat (limited to 'main.go')
| -rw-r--r-- | main.go | 105 |
1 files changed, 45 insertions, 60 deletions
@@ -2,16 +2,13 @@ package main import ( "flag" - "io/fs" "log/slog" "os" - "path/filepath" - "strings" "sync" "github.com/SayaAndy/saya-today-article-metadata-add/config" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/draft" - "github.com/SayaAndy/saya-today-article-metadata-add/internal/transcoder" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage" ) var configPath = flag.String("c", "config.json", "Path to the configuration file") @@ -19,8 +16,8 @@ var configPath = flag.String("c", "config.json", "Path to the configuration file func main() { flag.Parse() - cfg, err := config.InitConfig(*configPath) - if err != nil { + cfg := &config.Config{} + if err := config.LoadConfig(*configPath, cfg); err != nil { slog.Error("fail to load configuration", slog.String("error", err.Error())) os.Exit(1) } @@ -28,85 +25,73 @@ func main() { slog.SetLogLoggerLevel(cfg.LogLevel) slog.Info("starting metadata extractor...") - transcoders := make([]transcoder.Transcoder, 0, len(cfg.Transcoders)) - for _, tcCfg := range cfg.Transcoders { - newTranscoder, ok := transcoder.NewTranscoderMap[tcCfg.Type] - if !ok { - slog.Error("unsupported transcoder type", slog.String("type", tcCfg.Type)) - os.Exit(1) - } - t, err := newTranscoder(tcCfg.Config) - if err != nil { - slog.Error("fail to initialize transcoder", slog.String("type", tcCfg.Type), slog.String("error", err.Error())) - os.Exit(1) - } - transcoders = append(transcoders, t) - slog.Info("initialized transcoder", slog.String("type", tcCfg.Type)) + storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage, &cfg.DraftMode) + if err != nil { + slog.Error("fail to initialize input client", slog.String("error", err.Error())) + os.Exit(1) } - var drafts []string - err = filepath.WalkDir(cfg.DraftDir, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - return nil - } - if strings.HasSuffix(path, cfg.DraftSuffix) { - drafts = append(drafts, path) - } - return nil - }) + generalLogger := slog.With( + slog.String("storage_type", cfg.Storage.Type), + ) + generalLogger.Info("initialized storage client") + + files, err := storageClient.Scan() if err != nil { - slog.Error("fail to scan draft directory", slog.String("dir", cfg.DraftDir), slog.String("error", err.Error())) + generalLogger.Error("fail to scan input files", slog.String("error", err.Error())) os.Exit(1) } - slog.Info("scanned drafts", slog.Int("draft_count", len(drafts))) + generalLogger.Info("scanned files", slog.Int("file_count", len(files))) semaphore := make(chan struct{}, cfg.MaxConcurrentJobs) var wg sync.WaitGroup - wg.Add(len(drafts)) + wg.Add(len(files)) - for _, draftPath := range drafts { + for i, file := range files { semaphore <- struct{}{} - go func(path string) { + go func(index int, inputName string) { defer wg.Done() defer func() { <-semaphore }() + if cfg.DraftMode.Enabled && !storageClient.CompareDraftAndProd(inputName) { + generalLogger.Debug("skip a draft because prod object is identical to it", slog.String("file", inputName)) + return + } + generalLogger.Debug("processing a file", slog.String("file", inputName)) - codename := strings.TrimSuffix(filepath.Base(path), cfg.DraftSuffix) - fileLogger := slog.With(slog.String("draft", path), slog.String("codename", codename)) + reader, sz, err := storageClient.GetReader(inputName) + if err != nil { + generalLogger.Warn("fail to get reader for a file", slog.String("file", inputName), slog.String("error", err.Error())) + return + } + defer reader.Close() - content, err := os.ReadFile(path) + content := make([]byte, sz) + ln, err := reader.Read(content) if err != nil { - fileLogger.Warn("fail to read draft", slog.String("error", err.Error())) + generalLogger.Warn("fail to read content from a file", slog.String("file", inputName), slog.String("error", err.Error())) return } + generalLogger.Debug("read content from a file", + slog.String("file", inputName), + slog.Int64("expected_size", sz), + slog.Int("output_size", ln)) - doc, err := draft.ParseDraft(path, codename, content) + metadata, _, err := frontmatter.ParseFrontmatter(content) if err != nil { - fileLogger.Warn("fail to parse draft", slog.String("error", err.Error())) + generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", inputName), slog.String("error", err.Error())) return } - if doc.Metadata == nil { - fileLogger.Info("skip draft without frontmatter metadata") + + if metadata == nil { + generalLogger.Info("skip a file due to it not having metadata", slog.String("file", inputName)) return } - for _, t := range transcoders { - if err := t.Transcode(doc); err != nil { - fileLogger.Warn("transcoder failed", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) - } + if err = storageClient.WriteMetadata(inputName, metadata); err != nil { + generalLogger.Warn("fail to write metadata to a file", slog.String("file", inputName), slog.String("error", err.Error())) } - }(draftPath) + }(i, file) } wg.Wait() - - for _, t := range transcoders { - if err := t.Finalize(); err != nil { - slog.Error("fail to finalize transcoder", slog.String("transcoder", t.Name()), slog.String("error", err.Error())) - os.Exit(1) - } - slog.Info("finalized transcoder", slog.String("transcoder", t.Name())) - } } |