diff options
| -rw-r--r-- | config/config.go | 8 | ||||
| -rw-r--r-- | config/config.json | 2 | ||||
| -rw-r--r-- | internal/processor/webp.go | 2 | ||||
| -rw-r--r-- | main.go | 81 |
4 files changed, 55 insertions, 38 deletions
diff --git a/config/config.go b/config/config.go index 5df29df..0b6c4ff 100644 --- a/config/config.go +++ b/config/config.go @@ -9,9 +9,11 @@ import ( ) type Config struct { - Input InputConfig `json:"Input" validate:"required"` - Processor ProcessorConfig `json:"Processor" validate:"required"` - Output OutputConfig `json:"Output" validate:"required"` + Input InputConfig `json:"Input" validate:"required"` + Processor ProcessorConfig `json:"Processor" validate:"required"` + Output OutputConfig `json:"Output" validate:"required"` + MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"` + ForceRewrite bool `json:"ForceRewrite" validate:"required"` } type InputConfig struct { diff --git a/config/config.json b/config/config.json index 01b7727..742949f 100644 --- a/config/config.json +++ b/config/config.json @@ -1,4 +1,6 @@ { + "ForceRewrite": false, + "MaxConcurrentJobs": 4, "Input": { "Storage": { "Type": "b2", diff --git a/internal/processor/webp.go b/internal/processor/webp.go index b5e089d..86ff567 100644 --- a/internal/processor/webp.go +++ b/internal/processor/webp.go @@ -88,7 +88,7 @@ func (p *WebpProcessor) Process(contentType string, reader io.ReadCloser, writer minCoef = yCoef } - dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef), int(float64(src.Bounds().Max.Y)*minCoef))) + dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5))) draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil) return webp.Encode(writer, dst, opts) @@ -3,6 +3,7 @@ package main import ( "flag" "log/slog" + "sync" "github.com/SayaAndy/saya-today-thumbnail-generator/config" "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" @@ -18,7 +19,7 @@ func main() { flag.Parse() slog.Info("starting thumbnail generator...") - slog.SetLogLoggerLevel(slog.LevelDebug) + // slog.SetLogLoggerLevel(slog.LevelDebug) cfg := &config.Config{} if err := config.LoadConfig(*configPath, cfg); err != nil { @@ -51,49 +52,61 @@ func main() { if err != nil { panic(err) } + generalLogger.Info("scanned files", slog.Int("file_count", len(files))) - for _, file := range files { - outputName := converter.DeductOutputPath(file) - fileLogger := generalLogger.With(slog.String("input_path", file), slog.String("output_path", outputName)) + semaphore := make(chan struct{}, cfg.MaxConcurrentJobs) + var wg sync.WaitGroup + wg.Add(len(files)) - isMissing := outputClient.IsMissing(outputName) + for i, file := range files { + go func(index int, inputName string) { + semaphore <- struct{}{} + defer func() { <-semaphore }() + defer wg.Done() - inputMetadata, err := inputClient.ReadMetadata(file) - if err != nil { - fileLogger.Error("fail to read metadata of (supposedly existing) input file", slog.String("error", err.Error())) - continue - } + outputName := converter.DeductOutputPath(inputName) + fileLogger := generalLogger.With(slog.String("input_path", inputName), slog.String("output_path", outputName), slog.Int("file_index", index)) - if !isMissing { - outputMetadata, err := outputClient.ReadMetadata(outputName) + inputMetadata, err := inputClient.ReadMetadata(inputName) if err != nil { - fileLogger.Error("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error())) - continue + fileLogger.Error("fail to read metadata of (supposedly existing) input file", slog.String("error", err.Error())) + return } - if inputMetadata.Hash == outputMetadata.HashOriginal { - fileLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash)) - continue + + if !cfg.ForceRewrite && !outputClient.IsMissing(outputName) { + outputMetadata, err := outputClient.ReadMetadata(outputName) + if err != nil { + fileLogger.Error("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error())) + return + } + if inputMetadata.Hash == outputMetadata.HashOriginal { + fileLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash)) + return + } } - } - fileLogger.Info("start to process file") - reader, err := inputClient.GetReader(file) - if err != nil { - fileLogger.Error("fail to get reader for input file", slog.String("error", err.Error())) - continue - } + fileLogger.Info("start to process file") + reader, err := inputClient.GetReader(inputName) + if err != nil { + fileLogger.Error("fail to get reader for input file", slog.String("error", err.Error())) + return + } - writer, err := outputClient.GetWriter(outputName, inputMetadata) - if err != nil { - fileLogger.Error("fail to get writer for output file", slog.String("error", err.Error())) - continue - } + writer, err := outputClient.GetWriter(outputName, inputMetadata) + if err != nil { + fileLogger.Error("fail to get writer for output file", slog.String("error", err.Error())) + return + } - if err := converter.Process(inputMetadata.ContentType, reader, writer); err != nil { - fileLogger.Error("fail to convert file", slog.String("error", err.Error())) - continue - } + if err := converter.Process(inputMetadata.ContentType, reader, writer); err != nil { + fileLogger.Error("fail to convert file", slog.String("error", err.Error())) + return + } - fileLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash)) + fileLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash)) + }(i, file) } + + wg.Wait() + generalLogger.Info("all files processed successfully, exiting") } |