feat: implement concurrency

1 files changed, 47 insertions, 34 deletions

diff --git a/main.go b/main.go
index 87f30f5..9d8b625 100644
--- a/main.go
+++ b/main.go
@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"log/slog"
+	"sync"
 
 	"github.com/SayaAndy/saya-today-thumbnail-generator/config"
 	"github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
@@ -18,7 +19,7 @@ func main() {
 	flag.Parse()
 
 	slog.Info("starting thumbnail generator...")
-	slog.SetLogLoggerLevel(slog.LevelDebug)
+	// slog.SetLogLoggerLevel(slog.LevelDebug)
 
 	cfg := &config.Config{}
 	if err := config.LoadConfig(*configPath, cfg); err != nil {
@@ -51,49 +52,61 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
+	generalLogger.Info("scanned files", slog.Int("file_count", len(files)))
 
-	for _, file := range files {
-		outputName := converter.DeductOutputPath(file)
-		fileLogger := generalLogger.With(slog.String("input_path", file), slog.String("output_path", outputName))
+	semaphore := make(chan struct{}, cfg.MaxConcurrentJobs)
+	var wg sync.WaitGroup
+	wg.Add(len(files))
 
-		isMissing := outputClient.IsMissing(outputName)
+	for i, file := range files {
+		go func(index int, inputName string) {
+			semaphore <- struct{}{}
+			defer func() { <-semaphore }()
+			defer wg.Done()
 
-		inputMetadata, err := inputClient.ReadMetadata(file)
-		if err != nil {
-			fileLogger.Error("fail to read metadata of (supposedly existing) input file", slog.String("error", err.Error()))
-			continue
-		}
+			outputName := converter.DeductOutputPath(inputName)
+			fileLogger := generalLogger.With(slog.String("input_path", inputName), slog.String("output_path", outputName), slog.Int("file_index", index))
 
-		if !isMissing {
-			outputMetadata, err := outputClient.ReadMetadata(outputName)
+			inputMetadata, err := inputClient.ReadMetadata(inputName)
 			if err != nil {
-				fileLogger.Error("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error()))
-				continue
+				fileLogger.Error("fail to read metadata of (supposedly existing) input file", slog.String("error", err.Error()))
+				return
 			}
-			if inputMetadata.Hash == outputMetadata.HashOriginal {
-				fileLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash))
-				continue
+
+			if !cfg.ForceRewrite && !outputClient.IsMissing(outputName) {
+				outputMetadata, err := outputClient.ReadMetadata(outputName)
+				if err != nil {
+					fileLogger.Error("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error()))
+					return
+				}
+				if inputMetadata.Hash == outputMetadata.HashOriginal {
+					fileLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash))
+					return
+				}
 			}
-		}
 
-		fileLogger.Info("start to process file")
-		reader, err := inputClient.GetReader(file)
-		if err != nil {
-			fileLogger.Error("fail to get reader for input file", slog.String("error", err.Error()))
-			continue
-		}
+			fileLogger.Info("start to process file")
+			reader, err := inputClient.GetReader(inputName)
+			if err != nil {
+				fileLogger.Error("fail to get reader for input file", slog.String("error", err.Error()))
+				return
+			}
 
-		writer, err := outputClient.GetWriter(outputName, inputMetadata)
-		if err != nil {
-			fileLogger.Error("fail to get writer for output file", slog.String("error", err.Error()))
-			continue
-		}
+			writer, err := outputClient.GetWriter(outputName, inputMetadata)
+			if err != nil {
+				fileLogger.Error("fail to get writer for output file", slog.String("error", err.Error()))
+				return
+			}
 
-		if err := converter.Process(inputMetadata.ContentType, reader, writer); err != nil {
-			fileLogger.Error("fail to convert file", slog.String("error", err.Error()))
-			continue
-		}
+			if err := converter.Process(inputMetadata.ContentType, reader, writer); err != nil {
+				fileLogger.Error("fail to convert file", slog.String("error", err.Error()))
+				return
+			}
 
-		fileLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash))
+			fileLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash))
+		}(i, file)
 	}
+
+	wg.Wait()
+	generalLogger.Info("all files processed successfully, exiting")
 }