Diffstat (limited to 'config')
| -rw-r--r-- | config/config-b2.sample.json | 141 | ||||
| -rw-r--r-- | config/config-local-unix.sample.json | 41 | ||||
| -rw-r--r-- | config/config-s3.sample.json | 53 | ||||
| -rw-r--r-- | config/config.go | 237 |
4 files changed, 472 insertions, 0 deletions
diff --git a/config/config-b2.sample.json b/config/config-b2.sample.json new file mode 100644 index 0000000..60022dd --- /dev/null +++ b/config/config-b2.sample.json @@ -0,0 +1,141 @@ +{ + "MaxProcessThreads": 4, + "MaxPreProcessThreads": 12, + "LogLevel": "info", + "Input": { + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "full/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + }, + "KnownExtensions": [ + "jpg", + "jpeg", + "png" + ], + "CacheProcessed": true, + "CacheProcessedCsvPath": "cache.csv" + }, + "Converters": [ + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 320, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-320p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 560, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-560p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 800, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-800p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 1200, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-1200p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 1600, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-1600p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + } + ] +}
\ No newline at end of file diff --git a/config/config-local-unix.sample.json b/config/config-local-unix.sample.json new file mode 100644 index 0000000..941d995 --- /dev/null +++ b/config/config-local-unix.sample.json @@ -0,0 +1,41 @@ +{ + "MaxProcessThreads": 4, + "MaxPreProcessThreads": 12, + "LogLevel": "debug", + "Input": { + "Storage": { + "Type": "local-unix", + "Config": { + "MaxDepth": 3, + "Path": "/tmp/thumbnailing/full/" + } + }, + "KnownExtensions": [ + "jpg", + "jpeg", + "png" + ] + }, + "Converter": { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 800, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "local-unix", + "Config": { + "Path": "/tmp/thumbnailing/thumbnails/", + "DirPermissionMode": "0755", + "FilePermissionMode": "0644", + "AttributesImplementation": "xattr" + } + } + } + } +}
\ No newline at end of file diff --git a/config/config-s3.sample.json b/config/config-s3.sample.json new file mode 100644 index 0000000..5b067a5 --- /dev/null +++ b/config/config-s3.sample.json @@ -0,0 +1,53 @@ +{ + "MaxProcessThreads": 4, + "MaxPreProcessThreads": 12, + "LogLevel": "info", + "Input": { + "Storage": { + "Type": "s3", + "Config": { + "BucketName": "my-photos", + "Region": "", + "Prefix": "full/", + "Endpoint": "https://minio.local", + "UsePathStyle": true, + "AccessKeyID": "${AWS_ACCESS_KEY_ID}", + "SecretAccessKey": "${AWS_SECRET_ACCESS_KEY}" + } + }, + "KnownExtensions": [ + "jpg", + "jpeg", + "png" + ], + "CacheProcessed": true, + "CacheProcessedCsvPath": "cache.csv" + }, + "Converters": [ + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 320, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "s3", + "Config": { + "BucketName": "my-photos", + "Region": "", + "Prefix": "webp-320p/", + "Endpoint": "https://minio.local", + "UsePathStyle": true, + "AccessKeyID": "${AWS_ACCESS_KEY_ID}", + "SecretAccessKey": "${AWS_SECRET_ACCESS_KEY}" + } + } + } + } + ] +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..06fa94f --- /dev/null +++ b/config/config.go @@ -0,0 +1,237 @@ +package config + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + + "github.com/go-playground/validator/v10" +) + +type Config struct { + Input InputConfig `json:"Input" validate:"required"` + Converters []ConverterConfig `json:"Converters" validate:"required"` + MaxProcessThreads int `json:"MaxProcessThreads" validate:"required,min=1"` + MaxPreProcessThreads int `json:"MaxPreProcessThreads" validate:"min=1;gtefield=MaxProcessThreads"` + LogLevel slog.Level `json:"LogLevel" validate:"required"` +} + +type InputConfig struct { + Storage InputStorageConfig `json:"Storage" validate:"required"` + KnownExtensions []string `json:"KnownExtensions" validate:"required,min=0,dive,min=1"` + CacheProcessed bool `json:"CacheProcessed"` + CacheProcessedCsvPath string `json:"CacheProcessedCsvPath" validate:"filepath"` +} + +type InputStorageConfig struct { + Type string `json:"Type" validate:"required,oneof=b2 s3 local-unix"` + Config any `json:"Config" validate:"required"` +} + +func (sc *InputStorageConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + sc.Type = tmp.Type + + switch tmp.Type { + case "b2": + var b2Config B2Config + if err := json.Unmarshal(tmp.Config, &b2Config); err != nil { + return fmt.Errorf("unmarshal B2Config: %w", err) + } + sc.Config = &b2Config + case "s3": + var s3Config S3Config + if err := json.Unmarshal(tmp.Config, &s3Config); err != nil { + return fmt.Errorf("unmarshal S3Config: %w", err) + } + sc.Config = &s3Config + case "local-unix": + var localUnixConfig InputLocalUnixConfig + if err := json.Unmarshal(tmp.Config, &localUnixConfig); err != nil { + return fmt.Errorf("unmarshal LocalUnixConfig: %w", err) + } + sc.Config = &localUnixConfig + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type ConverterConfig struct { + Type string `json:"Type" validate:"required,oneof=webp jpeg"` + Config any `json:"Config" validate:"required"` + Output OutputConfig `json:"Output" validate:"required"` +} + +func (pc *ConverterConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + Output OutputConfig `json:"Output"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + pc.Type = tmp.Type + pc.Output = tmp.Output + + switch tmp.Type { + case "webp": + var webpConfig WebpConfig + if err := json.Unmarshal(tmp.Config, &webpConfig); err != nil { + return fmt.Errorf("unmarshal WebpConfig: %w", err) + } + pc.Config = &webpConfig + case "jpeg": + var jpegConfig JpegConfig + if err := json.Unmarshal(tmp.Config, &jpegConfig); err != nil { + return fmt.Errorf("unmarshal JpegConfig: %w", err) + } + pc.Config = &jpegConfig + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type WebpConfig struct { + Quality int `json:"Quality" validate:"required,min=1,max=100"` + Size SizeConfig `json:"Size"` +} + +type JpegConfig struct { + ExtensionName string `json:"ExtensionName" validate:"alpha"` + Quality int `json:"Quality" validate:"required,min=1,max=100"` + Size SizeConfig `json:"Size"` +} + +type SizeConfig struct { + MaxWidth int `json:"MaxWidth"` + MaxHeight int `json:"MaxHeight"` +} + +type OutputStorageConfig struct { + Type string `json:"Type" validate:"required,oneof=b2 s3 local-unix"` + Config any `json:"Config" validate:"required"` +} + +func (sc *OutputStorageConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + sc.Type = tmp.Type + + switch tmp.Type { + case "b2": + var b2Config B2Config + if err := json.Unmarshal(tmp.Config, &b2Config); err != nil { + return fmt.Errorf("unmarshal B2Config: %w", err) + } + sc.Config = &b2Config + case "s3": + var s3Config S3Config + if err := json.Unmarshal(tmp.Config, &s3Config); err != nil { + return fmt.Errorf("unmarshal S3Config: %w", err) + } + sc.Config = &s3Config + case "local-unix": + var localUnixConfig OutputLocalUnixConfig + if err := json.Unmarshal(tmp.Config, &localUnixConfig); err != nil { + return fmt.Errorf("unmarshal LocalUnixConfig: %w", err) + } + sc.Config = &localUnixConfig + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type B2Config struct { + BucketName string `json:"BucketName" validate:"required,min=1"` + Region string `json:"Region" validate:"required,min=1"` + Prefix string `json:"Prefix"` + KeyID string `json:"KeyID"` + ApplicationKey string `json:"ApplicationKey"` +} + +type S3Config struct { + BucketName string `json:"BucketName" validate:"required,min=1"` + Region string `json:"Region" validate:"required,min=1"` + Prefix string `json:"Prefix"` + Endpoint string `json:"Endpoint"` + UsePathStyle bool `json:"UsePathStyle"` + AccessKeyID string `json:"AccessKeyID"` + SecretAccessKey string `json:"SecretAccessKey"` +} + +type InputLocalUnixConfig struct { + MaxDepth int `json:"MaxDepth" validate:"required,min=0"` + Path string `json:"Path" validate:"required,min=1"` +} + +type OutputConfig struct { + RewriteOn string `json:"RewriteOn" validate:"oneof=Never UnequalHashInCache Always"` + Storage OutputStorageConfig `json:"Storage" validate:"required"` +} + +type OutputLocalUnixConfig struct { + Path string `json:"Path" validate:"required,min=1,dirpath"` + DirPermissionMode string `json:"DirPermissionMode" validate:"required,min=3"` + FilePermissionMode string `json:"FilePermissionMode" validate:"required,min=3"` + AttributesImplementation string `json:"AttributesImplementation" validate:"required,oneof=xattr none"` +} + +func LoadConfig(path string, config *Config) error { + fileBytes, err := os.ReadFile(path) + if err != nil { + return err + } + + expandedFileBytes := []byte(os.ExpandEnv(string(fileBytes))) + + if err = json.Unmarshal(expandedFileBytes, config); err != nil { + return err + } + + for i := range config.Converters { + if config.Converters[i].Output.RewriteOn == "" { + config.Converters[i].Output.RewriteOn = "UnequalHashInCache" + } + } + + return nil +} + +func InitConfig(path string) (*Config, error) { + config := &Config{} + if err := LoadConfig(path, config); err != nil { + return nil, err + } + + validate := validator.New(validator.WithRequiredStructEnabled()) + if err := validate.Struct(config); err != nil { + return nil, err + } + + return config, nil +} |