diff options
| author | 2025-08-19 00:14:30 +0700 | |
|---|---|---|
| committer | 2025-08-19 00:14:30 +0700 | |
| commit | 0b4bbb18087fe7e280a302f031e19852ef741a48 (patch) | |
| tree | 9e3fe40d0d17a022450b119f810f5405e9c51b90 | |
| parent | c9b2f8ca2239d64eaede7a0d755f038fe506bcca (diff) | |
| download | thumbnail-generator-0b4bbb18087fe7e280a302f031e19852ef741a48.tar.gz thumbnail-generator-0b4bbb18087fe7e280a302f031e19852ef741a48.zip | |
feat: allow multiple converters for processing
| -rw-r--r-- | config/config-b2.sample.json | 126 | ||||
| -rw-r--r-- | config/config-local-unix.sample.json | 20 | ||||
| -rw-r--r-- | config/config.go | 98 | ||||
| -rw-r--r-- | internal/client/input/b2.go | 3 | ||||
| -rw-r--r-- | internal/client/input/input_client_interface.go | 1 | ||||
| -rw-r--r-- | internal/client/input/local_unix.go | 7 | ||||
| -rw-r--r-- | internal/client/output/b2.go | 3 | ||||
| -rw-r--r-- | internal/client/output/local_unix.go | 7 | ||||
| -rw-r--r-- | internal/client/output/output_client_interface.go | 3 | ||||
| -rw-r--r-- | internal/converter/converter_interface.go | 6 | ||||
| -rw-r--r-- | internal/converter/webp.go | 52 | ||||
| -rw-r--r-- | main.go | 97 |
12 files changed, 273 insertions, 150 deletions
diff --git a/config/config-b2.sample.json b/config/config-b2.sample.json index c48b1b2..043c1a6 100644 --- a/config/config-b2.sample.json +++ b/config/config-b2.sample.json @@ -20,26 +20,116 @@ "png" ] }, - "Converter": { - "Type": "webp", - "Config": { - "Quality": 80, - "Size": { - "MaxWidth": 800, - "MaxHeight": 0 + "Converters": [ + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 320, + "MaxHeight": 0 + } + }, + "Output": { + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-320p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } } - } - }, - "Output": { - "Storage": { - "Type": "b2", + }, + { + "Type": "webp", "Config": { - "BucketName": "sayana-photos", - "Region": "eu-central-003", - "Prefix": "thumbnails/", - "KeyID": "${B2_KEY_ID}", - "ApplicationKey": "${B2_APPLICATION_KEY}" + "Quality": 80, + "Size": { + "MaxWidth": 560, + "MaxHeight": 0 + } + }, + "Output": { + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-560p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 800, + "MaxHeight": 0 + } + }, + "Output": { + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-800p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 1200, + "MaxHeight": 0 + } + }, + "Output": { + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-1200p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 1600, + "MaxHeight": 0 + } + }, + "Output": { + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-1600p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } } } - } + ] }
\ No newline at end of file diff --git a/config/config-local-unix.sample.json b/config/config-local-unix.sample.json index 36d8114..97f69f1 100644 --- a/config/config-local-unix.sample.json +++ b/config/config-local-unix.sample.json @@ -25,16 +25,16 @@ "MaxWidth": 800, "MaxHeight": 0 } - } - }, - "Output": { - "Storage": { - "Type": "local-unix", - "Config": { - "Path": "/tmp/thumbnailing/thumbnails/", - "DirPermissionMode": "0755", - "FilePermissionMode": "0644", - "AttributesImplementation": "xattr" + }, + "Output": { + "Storage": { + "Type": "local-unix", + "Config": { + "Path": "/tmp/thumbnailing/thumbnails/", + "DirPermissionMode": "0755", + "FilePermissionMode": "0644", + "AttributesImplementation": "xattr" + } } } } diff --git a/config/config.go b/config/config.go index 5b05ce9..ab051b2 100644 --- a/config/config.go +++ b/config/config.go @@ -10,13 +10,12 @@ import ( ) type Config struct { - Input InputConfig `json:"Input" validate:"required"` - Converter ConverterConfig `json:"Converter" validate:"required"` - Output OutputConfig `json:"Output" validate:"required"` - MaxProcessThreads int `json:"MaxProcessThreads" validate:"required,min=1"` - MaxPreProcessThreads int `json:"MaxPreProcessThreads" validate:"min=1;gtefield=MaxProcessThreads"` - ForceRewrite bool `json:"ForceRewrite" validate:"required"` - LogLevel slog.Level `json:"LogLevel" validate:"required"` + Input InputConfig `json:"Input" validate:"required"` + Converters []ConverterConfig `json:"Converters" validate:"required"` + MaxProcessThreads int `json:"MaxProcessThreads" validate:"required,min=1"` + MaxPreProcessThreads int `json:"MaxPreProcessThreads" validate:"min=1;gtefield=MaxProcessThreads"` + ForceRewrite bool `json:"ForceRewrite" validate:"required"` + LogLevel slog.Level `json:"LogLevel" validate:"required"` } type InputConfig struct { @@ -61,6 +60,50 @@ func (sc *InputStorageConfig) UnmarshalJSON(data []byte) error { return nil } +type ConverterConfig struct { + Type string `json:"Type" validate:"required,oneof=webp"` + Config any `json:"Config" validate:"required"` + Output OutputConfig `json:"Output" validate:"required"` +} + +func (pc *ConverterConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + Output OutputConfig `json:"Output"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + pc.Type = tmp.Type + pc.Output = tmp.Output + + switch tmp.Type { + case "webp": + var webpConfig WebpConfig + if err := json.Unmarshal(tmp.Config, &webpConfig); err != nil { + return fmt.Errorf("unmarshal WebpConfig: %w", err) + } + pc.Config = &webpConfig + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type WebpConfig struct { + Quality int `json:"Quality" validate:"required,min=1,max=100"` + Size SizeConfig `json:"Size" validate:"required"` +} + +type SizeConfig struct { + MaxWidth int `json:"MaxWidth" validate:"required,min=0"` + MaxHeight int `json:"MaxHeight" validate:"required,min=0"` +} + type OutputStorageConfig struct { Type string `json:"Type" validate:"required,oneof=b2 local-unix"` Config any `json:"Config" validate:"required"` @@ -122,47 +165,6 @@ type OutputLocalUnixConfig struct { AttributesImplementation string `json:"AttributesImplementation" validate:"required,oneof=xattr none"` } -type ConverterConfig struct { - Type string `json:"Type" validate:"required,oneof=webp"` - Config any `json:"Config" validate:"required"` -} - -func (pc *ConverterConfig) UnmarshalJSON(data []byte) error { - var tmp struct { - Type string `json:"Type"` - Config json.RawMessage `json:"Config"` - } - - if err := json.Unmarshal(data, &tmp); err != nil { - return err - } - - pc.Type = tmp.Type - - switch tmp.Type { - case "webp": - var webpConfig WebpConfig - if err := json.Unmarshal(tmp.Config, &webpConfig); err != nil { - return fmt.Errorf("unmarshal WebpConfig: %w", err) - } - pc.Config = &webpConfig - default: - return fmt.Errorf("unsupported storage type: %s", tmp.Type) - } - - return nil -} - -type WebpConfig struct { - Quality int `json:"Quality" validate:"required,min=1,max=100"` - Size SizeConfig `json:"Size" validate:"required"` -} - -type SizeConfig struct { - MaxWidth int `json:"MaxWidth" validate:"required,min=0"` - MaxHeight int `json:"MaxHeight" validate:"required,min=0"` -} - func LoadConfig(path string, config *Config) error { fileBytes, err := os.ReadFile(path) if err != nil { diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go index 0092313..db68268 100644 --- a/internal/client/input/b2.go +++ b/internal/client/input/b2.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "slices" - "strconv" "strings" "github.com/Backblaze/blazer/b2" @@ -102,8 +101,8 @@ func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) { FirstCreated: attrs.UploadTimestamp, LastModified: attrs.LastModified, Misc: attrs.Info, + Size: attrs.Size, } - metadata.Misc["Size"] = strconv.FormatInt(attrs.Size, 10) switch attrs.Status { case b2.Uploaded: diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go index 5d12fa8..4930aa0 100644 --- a/internal/client/input/input_client_interface.go +++ b/internal/client/input/input_client_interface.go @@ -20,6 +20,7 @@ type MetadataStruct struct { ContentType string FirstCreated time.Time LastModified time.Time + Size int64 Misc map[string]string } diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go index 2df3a41..27a7947 100644 --- a/internal/client/input/local_unix.go +++ b/internal/client/input/local_unix.go @@ -87,10 +87,6 @@ func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error stat_t := fileInfo.Sys().(*syscall.Stat_t) creationTime := time.Unix(stat_t.Ctim.Sec, stat_t.Ctim.Nsec) - misc := map[string]string{ - "Size": strconv.FormatInt(fileInfo.Size(), 10), - } - return &MetadataStruct{ Name: fileInfo.Name(), StorageType: "local-unix", @@ -98,7 +94,8 @@ func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error ContentType: mime.TypeByExtension("." + nodeExt), FirstCreated: creationTime, LastModified: fileInfo.ModTime(), - Misc: misc, + Size: fileInfo.Size(), + Misc: map[string]string{}, }, nil } diff --git a/internal/client/output/b2.go b/internal/client/output/b2.go index e741e25..f53532a 100644 --- a/internal/client/output/b2.go +++ b/internal/client/output/b2.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "strconv" "github.com/Backblaze/blazer/b2" "github.com/SayaAndy/saya-today-thumbnail-generator/config" @@ -70,8 +69,8 @@ func (c *B2OutputClient) ReadMetadata(path string) (*MetadataStruct, error) { FirstCreated: attrs.UploadTimestamp, LastModified: attrs.LastModified, Misc: attrs.Info, + Size: attrs.Size, } - metadata.Misc["Size"] = strconv.FormatInt(attrs.Size, 10) switch attrs.Status { case b2.Uploaded: diff --git a/internal/client/output/local_unix.go b/internal/client/output/local_unix.go index b106726..dd53e79 100644 --- a/internal/client/output/local_unix.go +++ b/internal/client/output/local_unix.go @@ -93,10 +93,6 @@ func (c *LocalUnixOutputClient) ReadMetadata(path string) (*MetadataStruct, erro stat_t := fileInfo.Sys().(*syscall.Stat_t) creationTime := time.Unix(stat_t.Ctim.Sec, stat_t.Ctim.Nsec) - misc := map[string]string{ - "Size": strconv.FormatInt(fileInfo.Size(), 10), - } - mddateOriginal := make([]byte, 0) switch c.attrMode { case "xattr": @@ -122,7 +118,8 @@ func (c *LocalUnixOutputClient) ReadMetadata(path string) (*MetadataStruct, erro ContentType: mime.TypeByExtension("." + nodeExt), FirstCreated: creationTime, LastModified: fileInfo.ModTime(), - Misc: misc, + Size: fileInfo.Size(), + Misc: map[string]string{}, }, nil } diff --git a/internal/client/output/output_client_interface.go b/internal/client/output/output_client_interface.go index 437895c..c74851d 100644 --- a/internal/client/output/output_client_interface.go +++ b/internal/client/output/output_client_interface.go @@ -10,7 +10,7 @@ import ( type OutputClient interface { GetWriter(path string, inputMetadata *input.MetadataStruct) (io.WriteCloser, error) - ReadMetadata(string) (*MetadataStruct, error) + ReadMetadata(path string) (*MetadataStruct, error) IsMissing(path string) bool } @@ -22,6 +22,7 @@ type MetadataStruct struct { ContentType string FirstCreated time.Time LastModified time.Time + Size int64 Misc map[string]string } diff --git a/internal/converter/converter_interface.go b/internal/converter/converter_interface.go index 342ca1e..fda23d1 100644 --- a/internal/converter/converter_interface.go +++ b/internal/converter/converter_interface.go @@ -4,11 +4,15 @@ import ( "io" "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" ) type Converter interface { + Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error DeductOutputPath(inputPath string) string - Process(ext string, reader io.ReadCloser, writer io.WriteCloser) error + ReadMetadata(path string) (*output.MetadataStruct, error) + IsMissing(path string) bool } var NewConverterMap = map[string]func(cfg *config.ConverterConfig) (Converter, error){ diff --git a/internal/converter/webp.go b/internal/converter/webp.go index 8a9c09a..8cfe5a7 100644 --- a/internal/converter/webp.go +++ b/internal/converter/webp.go @@ -12,6 +12,8 @@ import ( "golang.org/x/image/draw" "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" "github.com/kolesa-team/go-webp/encoder" "github.com/kolesa-team/go-webp/webp" ) @@ -19,9 +21,10 @@ import ( var _ Converter = (*WebpConverter)(nil) type WebpConverter struct { - maxWidth int - maxHeight int - quality int + maxWidth int + maxHeight int + quality int + outputClient output.OutputClient } func NewWebpConverter(cfg *config.ConverterConfig) (Converter, error) { @@ -30,26 +33,24 @@ func NewWebpConverter(cfg *config.ConverterConfig) (Converter, error) { } webpCfg := cfg.Config.(*config.WebpConfig) - return &WebpConverter{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality}, nil -} - -func (p *WebpConverter) DeductOutputPath(inputPath string) string { - pathParts := strings.Split(inputPath, ".") - if len(pathParts) < 2 { - return inputPath + ".webp" + outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output) + if err != nil { + return nil, fmt.Errorf("fail to initialize output client: %w", err) } - pathParts[len(pathParts)-1] = "webp" - return strings.Join(pathParts, ".") + + return &WebpConverter{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality, outputClient}, nil } -func (p *WebpConverter) Process(contentType string, reader io.ReadCloser, writer io.WriteCloser) error { +func (p *WebpConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error { var src image.Image - var err error - defer reader.Close() + writer, err := p.outputClient.GetWriter(outputName, inputMetadata) + if err != nil { + return fmt.Errorf("fail to initialize writer for output: %w", err) + } defer writer.Close() - switch contentType { + switch inputMetadata.ContentType { case "image/jpeg": src, err = jpeg.Decode(reader) if err != nil { @@ -61,7 +62,7 @@ func (p *WebpConverter) Process(contentType string, reader io.ReadCloser, writer return fmt.Errorf("decode png: %w", err) } default: - return fmt.Errorf("unsupported content type: %s", contentType) + return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType) } opts, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, float32(p.quality)) @@ -93,3 +94,20 @@ func (p *WebpConverter) Process(contentType string, reader io.ReadCloser, writer return webp.Encode(writer, dst, opts) } + +func (p *WebpConverter) DeductOutputPath(inputPath string) string { + pathParts := strings.Split(inputPath, ".") + if len(pathParts) < 2 { + return inputPath + ".webp" + } + pathParts[len(pathParts)-1] = "webp" + return strings.Join(pathParts, ".") +} + +func (p *WebpConverter) ReadMetadata(path string) (*output.MetadataStruct, error) { + return p.outputClient.ReadMetadata(path) +} + +func (p *WebpConverter) IsMissing(path string) bool { + return p.outputClient.IsMissing(path) +} @@ -1,16 +1,17 @@ package main import ( + "bytes" "flag" "log/slog" "os" "os/signal" + "strings" "sync" "syscall" "github.com/SayaAndy/saya-today-thumbnail-generator/config" "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" "github.com/SayaAndy/saya-today-thumbnail-generator/internal/converter" ) @@ -46,24 +47,20 @@ func main() { os.Exit(1) } - outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output) - if err != nil { - slog.Error("fail to initialize output client", slog.String("error", err.Error())) - os.Exit(1) - } - - conv, err := converter.NewConverterMap[cfg.Converter.Type](&cfg.Converter) - if err != nil { - slog.Error("fail to initialize converter", slog.String("error", err.Error())) - os.Exit(1) + converters := make([]converter.Converter, 0, len(cfg.Converters)) + var converterTypes []string + for _, converterCfg := range cfg.Converters { + conv, err := converter.NewConverterMap[converterCfg.Type](&converterCfg) + if err != nil { + slog.Error("fail to initialize converter", slog.String("error", err.Error())) + os.Exit(1) + } + converters = append(converters, conv) + converterTypes = append(converterTypes, converterCfg.Type) } - generalLogger := slog.With( - slog.String("input_storage", cfg.Input.Storage.Type), - slog.String("converter_type", cfg.Converter.Type), - slog.String("output_storage", cfg.Output.Storage.Type), - ) - generalLogger.Info("initialized clients and converter") + generalLogger := slog.With(slog.String("input_storage", cfg.Input.Storage.Type)) + generalLogger.Info("initialized input client and converters", slog.String("converter_types", strings.Join(converterTypes, " "))) select { case <-sigTermChan: @@ -77,7 +74,8 @@ func main() { generalLogger.Error("fail to scan input files", slog.String("error", err.Error())) os.Exit(1) } - generalLogger.Info("scanned files", slog.Int("file_count", len(files))) + fileCount := len(files) + generalLogger.Info("scanned files", slog.Int("file_count", fileCount)) select { case <-sigTermChan: @@ -89,13 +87,12 @@ func main() { processSemaphore := make(chan struct{}, cfg.MaxProcessThreads) queueSemaphore := make(chan struct{}, cfg.MaxPreProcessThreads) var wg sync.WaitGroup - wg.Add(len(files)) + wg.Add(fileCount) for i, file := range files { queueSemaphore <- struct{}{} go func(index int, inputName string) { - outputName := conv.DeductOutputPath(inputName) - fileLogger := generalLogger.With(slog.String("input_path", inputName), slog.String("output_path", outputName), slog.Int("file_index", index)) + fileLogger := generalLogger.With(slog.String("input_path", inputName), slog.Int("file_index", index)) threadSigTermChannel := make(chan os.Signal, 1) signal.Notify(threadSigTermChannel, os.Interrupt, syscall.SIGTERM) @@ -115,42 +112,60 @@ func main() { return } - originalInputHash := "" - if !cfg.ForceRewrite && !outputClient.IsMissing(outputName) { - outputMetadata, err := outputClient.ReadMetadata(outputName) - if err != nil { - fileLogger.Warn("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error())) - return - } - originalInputHash = outputMetadata.HashOriginal - if inputMetadata.Hash == originalInputHash { - fileLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash)) - return + convertersToLaunch := []int{} + for j, conv := range converters { + outputName := conv.DeductOutputPath(inputName) + originalInputHash := "" + convLogger := fileLogger.With(slog.String("output_path", outputName), slog.Int("conv_index", j)) + + if !cfg.ForceRewrite && !conv.IsMissing(outputName) { + outputMetadata, err := conv.ReadMetadata(outputName) + if err != nil { + convLogger.Warn("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error())) + continue + } + originalInputHash = outputMetadata.HashOriginal + if inputMetadata.Hash == originalInputHash { + convLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash)) + continue + } } + convertersToLaunch = append(convertersToLaunch, j) + } + + if len(convertersToLaunch) == 0 { + return } processSemaphore <- struct{}{} defer func() { <-processSemaphore }() - fileLogger.Info("start to process file", slog.String("input_hash", inputMetadata.Hash), slog.String("original_input_hash", originalInputHash)) + fileLogger.Info("start to process file", slog.String("input_hash", inputMetadata.Hash)) reader, err := inputClient.GetReader(inputName) if err != nil { fileLogger.Warn("fail to get reader for input file", slog.String("error", err.Error())) return } - writer, err := outputClient.GetWriter(outputName, inputMetadata) - if err != nil { - fileLogger.Warn("fail to get writer for output file", slog.String("error", err.Error())) + fileContent := make([]byte, inputMetadata.Size) + if _, err = reader.Read(fileContent); err != nil { + fileLogger.Warn("fail to read content of input file", slog.String("error", err.Error())) return } + reader.Close() - if err := conv.Process(inputMetadata.ContentType, reader, writer); err != nil { - fileLogger.Warn("fail to convert file", slog.String("error", err.Error())) - return - } + for _, convIndex := range convertersToLaunch { + conv := converters[convIndex] + outputName := conv.DeductOutputPath(inputName) + convLogger := fileLogger.With(slog.String("output_path", outputName), slog.Int("conv_index", convIndex)) - fileLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash)) + if err := conv.Process(inputMetadata, bytes.NewReader(fileContent), outputName); err != nil { + convLogger.Warn("fail to convert file", slog.String("error", err.Error())) + return + } + + convLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash)) + } }(i, file) } |