| -rw-r--r-- | .gitignore | 10 | ||||
| -rw-r--r-- | CLAUDE.md | 42 | ||||
| -rw-r--r-- | config/config-b2.sample.json | 141 | ||||
| -rw-r--r-- | config/config-local-unix.sample.json | 41 | ||||
| -rw-r--r-- | config/config-s3.sample.json | 53 | ||||
| -rw-r--r-- | config/config.go | 172 | ||||
| -rw-r--r-- | config/config.json | 31 | ||||
| -rw-r--r-- | go.mod | 39 | ||||
| -rw-r--r-- | go.sum | 73 | ||||
| -rw-r--r-- | internal/client/input/b2.go | 135 | ||||
| -rw-r--r-- | internal/client/input/input_client_interface.go | 32 | ||||
| -rw-r--r-- | internal/client/input/local_unix.go | 108 | ||||
| -rw-r--r-- | internal/client/input/s3.go | 157 | ||||
| -rw-r--r-- | internal/client/output/b2.go | 113 | ||||
| -rw-r--r-- | internal/client/output/local_unix.go | 129 | ||||
| -rw-r--r-- | internal/client/output/output_client_interface.go | 33 | ||||
| -rw-r--r-- | internal/client/output/s3.go | 170 | ||||
| -rw-r--r-- | internal/converter/converter_interface.go | 21 | ||||
| -rw-r--r-- | internal/converter/jpeg.go | 114 | ||||
| -rw-r--r-- | internal/converter/webp.go | 118 | ||||
| -rw-r--r-- | main.go | 286 |
21 files changed, 1942 insertions, 76 deletions
@@ -26,7 +26,13 @@ go.work.sum # env file .env +.envrc # Editor/IDE -# .idea/ -# .vscode/ +.idea/ +.vscode/ + +config*.json +!config*.sample.json + +cache.csv diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..94be9b2 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,42 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build & Run + +```bash +go build -o saya-today-thumbnail-generator +./saya-today-thumbnail-generator -c config/config-local-unix.sample.json +``` + +No Makefile. No test files exist yet — `go test ./...` would be the convention. + +## What This Is + +Concurrent batch thumbnail generator. Reads images from a source (Backblaze B2, S3/S3-compatible, or local filesystem), converts to WebP/JPEG with configurable quality/size, writes to output storage. Maintains a CSV-based file cache to skip already-processed images. + +## Architecture + +Three pluggable abstractions, each using a factory/registry pattern keyed by string type: + +- **InputClient** (`internal/client/input/`) — scans source files, provides readers. Implementations: `b2`, `s3`, `local-unix` +- **OutputClient** (`internal/client/output/`) — writes converted files. Implementations: `b2`, `s3`, `local-unix` +- **Converter** (`internal/converter/`) — decodes image, resizes (Catmull-Rom), encodes to target format. Implementations: `webp`, `jpeg` + +Each has a `New*Map` factory function returning a map of type-string → constructor. + +## Config System (`config/config.go`) + +JSON config with discriminated unions — `"Type"` field selects which struct to unmarshal into. Supports `${ENV_VAR}` expansion in string values. Validated with go-playground/validator. + +Key config knobs: `MaxProcessThreads` (conversion concurrency), `MaxPreProcessThreads` (I/O concurrency), `RewriteOn` strategy per converter (`Never`/`UnequalHashInCache`/`Always`). + +Sample configs in `config/config-*.sample.json`. Actual configs are gitignored. + +## Concurrency Model + +Two-tier semaphore system in `main.go`: pre-process (file scanning/reading) and process (image conversion) run with separate thread pool limits. Graceful shutdown on SIGTERM/SIGINT via early-termination flag. Cache map protected by `sync.RWMutex`. + +## Environment + +Uses direnv (`.envrc`) for credentials (`B2_KEY_ID`, `B2_APPLICATION_KEY`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`). S3 client supports custom endpoints and path-style addressing for S3-compatible stores (MinIO, etc.). diff --git a/config/config-b2.sample.json b/config/config-b2.sample.json new file mode 100644 index 0000000..60022dd --- /dev/null +++ b/config/config-b2.sample.json @@ -0,0 +1,141 @@ +{ + "MaxProcessThreads": 4, + "MaxPreProcessThreads": 12, + "LogLevel": "info", + "Input": { + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "full/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + }, + "KnownExtensions": [ + "jpg", + "jpeg", + "png" + ], + "CacheProcessed": true, + "CacheProcessedCsvPath": "cache.csv" + }, + "Converters": [ + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 320, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-320p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 560, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-560p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 800, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-800p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 1200, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-1200p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + }, + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 1600, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-photos", + "Region": "eu-central-003", + "Prefix": "webp-1600p/", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } + } + } + ] +}
\ No newline at end of file diff --git a/config/config-local-unix.sample.json b/config/config-local-unix.sample.json new file mode 100644 index 0000000..941d995 --- /dev/null +++ b/config/config-local-unix.sample.json @@ -0,0 +1,41 @@ +{ + "MaxProcessThreads": 4, + "MaxPreProcessThreads": 12, + "LogLevel": "debug", + "Input": { + "Storage": { + "Type": "local-unix", + "Config": { + "MaxDepth": 3, + "Path": "/tmp/thumbnailing/full/" + } + }, + "KnownExtensions": [ + "jpg", + "jpeg", + "png" + ] + }, + "Converter": { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 800, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "local-unix", + "Config": { + "Path": "/tmp/thumbnailing/thumbnails/", + "DirPermissionMode": "0755", + "FilePermissionMode": "0644", + "AttributesImplementation": "xattr" + } + } + } + } +}
\ No newline at end of file diff --git a/config/config-s3.sample.json b/config/config-s3.sample.json new file mode 100644 index 0000000..5b067a5 --- /dev/null +++ b/config/config-s3.sample.json @@ -0,0 +1,53 @@ +{ + "MaxProcessThreads": 4, + "MaxPreProcessThreads": 12, + "LogLevel": "info", + "Input": { + "Storage": { + "Type": "s3", + "Config": { + "BucketName": "my-photos", + "Region": "", + "Prefix": "full/", + "Endpoint": "https://minio.local", + "UsePathStyle": true, + "AccessKeyID": "${AWS_ACCESS_KEY_ID}", + "SecretAccessKey": "${AWS_SECRET_ACCESS_KEY}" + } + }, + "KnownExtensions": [ + "jpg", + "jpeg", + "png" + ], + "CacheProcessed": true, + "CacheProcessedCsvPath": "cache.csv" + }, + "Converters": [ + { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 320, + "MaxHeight": 0 + } + }, + "Output": { + "RewriteOn": "UnequalHashInCache", + "Storage": { + "Type": "s3", + "Config": { + "BucketName": "my-photos", + "Region": "", + "Prefix": "webp-320p/", + "Endpoint": "https://minio.local", + "UsePathStyle": true, + "AccessKeyID": "${AWS_ACCESS_KEY_ID}", + "SecretAccessKey": "${AWS_SECRET_ACCESS_KEY}" + } + } + } + } + ] +} diff --git a/config/config.go b/config/config.go index 6d37904..06fa94f 100644 --- a/config/config.go +++ b/config/config.go @@ -3,32 +3,132 @@ package config import ( "encoding/json" "fmt" + "log/slog" "os" "github.com/go-playground/validator/v10" ) type Config struct { - Input InputConfig `json:"Input" validate:"required"` - Output OutputConfig `json:"Output" validate:"required"` + Input InputConfig `json:"Input" validate:"required"` + Converters []ConverterConfig `json:"Converters" validate:"required"` + MaxProcessThreads int `json:"MaxProcessThreads" validate:"required,min=1"` + MaxPreProcessThreads int `json:"MaxPreProcessThreads" validate:"min=1;gtefield=MaxProcessThreads"` + LogLevel slog.Level `json:"LogLevel" validate:"required"` } type InputConfig struct { - Storage StorageConfig `json:"Storage" validate:"required"` + Storage InputStorageConfig `json:"Storage" validate:"required"` + KnownExtensions []string `json:"KnownExtensions" validate:"required,min=0,dive,min=1"` + CacheProcessed bool `json:"CacheProcessed"` + CacheProcessedCsvPath string `json:"CacheProcessedCsvPath" validate:"filepath"` } -type OutputConfig struct { - Storage StorageConfig `json:"Storage" validate:"required"` - Quality int `json:"Quality" validate:"required,min=1,max=100"` - Size SizeConfig `json:"Size" validate:"required"` +type InputStorageConfig struct { + Type string `json:"Type" validate:"required,oneof=b2 s3 local-unix"` + Config any `json:"Config" validate:"required"` +} + +func (sc *InputStorageConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + sc.Type = tmp.Type + + switch tmp.Type { + case "b2": + var b2Config B2Config + if err := json.Unmarshal(tmp.Config, &b2Config); err != nil { + return fmt.Errorf("unmarshal B2Config: %w", err) + } + sc.Config = &b2Config + case "s3": + var s3Config S3Config + if err := json.Unmarshal(tmp.Config, &s3Config); err != nil { + return fmt.Errorf("unmarshal S3Config: %w", err) + } + sc.Config = &s3Config + case "local-unix": + var localUnixConfig InputLocalUnixConfig + if err := json.Unmarshal(tmp.Config, &localUnixConfig); err != nil { + return fmt.Errorf("unmarshal LocalUnixConfig: %w", err) + } + sc.Config = &localUnixConfig + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type ConverterConfig struct { + Type string `json:"Type" validate:"required,oneof=webp jpeg"` + Config any `json:"Config" validate:"required"` + Output OutputConfig `json:"Output" validate:"required"` +} + +func (pc *ConverterConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + Output OutputConfig `json:"Output"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + pc.Type = tmp.Type + pc.Output = tmp.Output + + switch tmp.Type { + case "webp": + var webpConfig WebpConfig + if err := json.Unmarshal(tmp.Config, &webpConfig); err != nil { + return fmt.Errorf("unmarshal WebpConfig: %w", err) + } + pc.Config = &webpConfig + case "jpeg": + var jpegConfig JpegConfig + if err := json.Unmarshal(tmp.Config, &jpegConfig); err != nil { + return fmt.Errorf("unmarshal JpegConfig: %w", err) + } + pc.Config = &jpegConfig + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type WebpConfig struct { + Quality int `json:"Quality" validate:"required,min=1,max=100"` + Size SizeConfig `json:"Size"` } -type StorageConfig struct { - Type string `json:"Type" validate:"required,oneof=b2 local"` +type JpegConfig struct { + ExtensionName string `json:"ExtensionName" validate:"alpha"` + Quality int `json:"Quality" validate:"required,min=1,max=100"` + Size SizeConfig `json:"Size"` +} + +type SizeConfig struct { + MaxWidth int `json:"MaxWidth"` + MaxHeight int `json:"MaxHeight"` +} + +type OutputStorageConfig struct { + Type string `json:"Type" validate:"required,oneof=b2 s3 local-unix"` Config any `json:"Config" validate:"required"` } -func (sc *StorageConfig) UnmarshalJSON(data []byte) error { +func (sc *OutputStorageConfig) UnmarshalJSON(data []byte) error { var tmp struct { Type string `json:"Type"` Config json.RawMessage `json:"Config"` @@ -46,13 +146,19 @@ func (sc *StorageConfig) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(tmp.Config, &b2Config); err != nil { return fmt.Errorf("unmarshal B2Config: %w", err) } - sc.Config = b2Config - case "local": - var localConfig LocalConfig - if err := json.Unmarshal(tmp.Config, &localConfig); err != nil { - return fmt.Errorf("unmarshal LocalConfig: %w", err) + sc.Config = &b2Config + case "s3": + var s3Config S3Config + if err := json.Unmarshal(tmp.Config, &s3Config); err != nil { + return fmt.Errorf("unmarshal S3Config: %w", err) } - sc.Config = localConfig + sc.Config = &s3Config + case "local-unix": + var localUnixConfig OutputLocalUnixConfig + if err := json.Unmarshal(tmp.Config, &localUnixConfig); err != nil { + return fmt.Errorf("unmarshal LocalUnixConfig: %w", err) + } + sc.Config = &localUnixConfig default: return fmt.Errorf("unsupported storage type: %s", tmp.Type) } @@ -68,13 +174,31 @@ type B2Config struct { ApplicationKey string `json:"ApplicationKey"` } -type LocalConfig struct { - Path string `json:"Path" validate:"required,min=1"` +type S3Config struct { + BucketName string `json:"BucketName" validate:"required,min=1"` + Region string `json:"Region" validate:"required,min=1"` + Prefix string `json:"Prefix"` + Endpoint string `json:"Endpoint"` + UsePathStyle bool `json:"UsePathStyle"` + AccessKeyID string `json:"AccessKeyID"` + SecretAccessKey string `json:"SecretAccessKey"` +} + +type InputLocalUnixConfig struct { + MaxDepth int `json:"MaxDepth" validate:"required,min=0"` + Path string `json:"Path" validate:"required,min=1"` } -type SizeConfig struct { - MaxWidth int `json:"MaxWidth" validate:"required,min=0"` - MaxHeight int `json:"MaxHeight" validate:"required,min=0"` +type OutputConfig struct { + RewriteOn string `json:"RewriteOn" validate:"oneof=Never UnequalHashInCache Always"` + Storage OutputStorageConfig `json:"Storage" validate:"required"` +} + +type OutputLocalUnixConfig struct { + Path string `json:"Path" validate:"required,min=1,dirpath"` + DirPermissionMode string `json:"DirPermissionMode" validate:"required,min=3"` + FilePermissionMode string `json:"FilePermissionMode" validate:"required,min=3"` + AttributesImplementation string `json:"AttributesImplementation" validate:"required,oneof=xattr none"` } func LoadConfig(path string, config *Config) error { @@ -89,6 +213,12 @@ func LoadConfig(path string, config *Config) error { return err } + for i := range config.Converters { + if config.Converters[i].Output.RewriteOn == "" { + config.Converters[i].Output.RewriteOn = "UnequalHashInCache" + } + } + return nil } diff --git a/config/config.json b/config/config.json deleted file mode 100644 index d818107..0000000 --- a/config/config.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "Input": { - "Storage": { - "Type": "b2", - "Config": { - "BucketName": "sayana-photos", - "Region": "eu-central-003", - "Prefix": "full/", - "KeyID": "${B2_KEY_ID}", - "ApplicationKey": "${B2_APPLICATION_KEY}" - } - } - }, - "Output": { - "Storage": { - "Type": "b2", - "Config": { - "BucketName": "sayana-photos", - "Region": "eu-central-003", - "Prefix": "thumbnails/", - "KeyID": "${B2_KEY_ID}", - "ApplicationKey": "${B2_APPLICATION_KEY}" - } - }, - "Quality": 80, - "Size": { - "MaxWidth": 1280, - "MaxHeight": 0 - } - } -}
\ No newline at end of file @@ -1,16 +1,39 @@ module github.com/SayaAndy/saya-today-thumbnail-generator -go 1.24.4 +go 1.26.0 require ( - github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/Backblaze/blazer v0.7.2 + github.com/go-playground/validator/v10 v10.30.1 + github.com/kolesa-team/go-webp v1.0.5 + golang.org/x/image v0.38.0 + golang.org/x/sys v0.42.0 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.14 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/smithy-go v1.24.2 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.27.0 // indirect - github.com/kolesa-team/go-webp v1.0.5 // indirect github.com/leodido/go-urn v1.4.0 // indirect - golang.org/x/crypto v0.33.0 // indirect - golang.org/x/net v0.34.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/crypto v0.49.0 // indirect + golang.org/x/text v0.35.0 // indirect ) @@ -1,26 +1,75 @@ +github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM= +github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= +github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= -github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/kolesa-team/go-webp v1.0.5 h1:GZQHJBaE8dsNKZltfwqsL0qVJ7vqHXsfA+4AHrQW3pE= github.com/kolesa-team/go-webp v1.0.5/go.mod h1:QmJu0YHXT3ex+4SgUvs+a+1SFCDcCqyZg+LbIuNNTnE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= +golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go new file mode 100644 index 0000000..fe94333 --- /dev/null +++ b/internal/client/input/b2.go @@ -0,0 +1,135 @@ +package input + +import ( + "context" + "fmt" + "io" + "slices" + "strings" + + "github.com/Backblaze/blazer/b2" + "github.com/SayaAndy/saya-today-thumbnail-generator/config" +) + +var _ InputClient = (*B2InputClient)(nil) + +type B2InputClient struct { + prefix string + bucket *b2.Bucket + bucketName string + b2cl *b2.Client + knownExtensions []string +} + +func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) { + if cfg.Storage.Type != "b2" { + return nil, fmt.Errorf("invalid storage type for B2InputClient") + } + b2cfg := cfg.Storage.Config.(*config.B2Config) + + b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) + if err != nil { + return nil, err + } + + bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) + if err != nil { + return nil, err + } + + return &B2InputClient{b2cl: b2cl, bucket: bucket, bucketName: b2cfg.BucketName, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil +} + +func (c *B2InputClient) Scan() ([]string, error) { + filePaths := []string{} + + iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix)) + + for iter.Next() { + obj := iter.Object() + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + if attrs.Status != b2.Uploaded { + continue + } + + name := obj.Name() + + if len(c.knownExtensions) != 0 { + nameParts := strings.Split(name, ".") + if len(nameParts) < 2 { + continue + } + ext := strings.ToLower(nameParts[len(nameParts)-1]) + if !slices.Contains(c.knownExtensions, ext) { + continue + } + } + + filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix)) + } + + if err := iter.Err(); err != nil { + return nil, fmt.Errorf("iterate over B2 objects: %w", err) + } + + return filePaths, nil +} + +func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return nil, fmt.Errorf("object not found in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + metadata := MetadataStruct{ + Name: attrs.Name, + StorageType: "b2", + Hash: attrs.SHA1, + ContentType: attrs.ContentType, + FirstCreated: attrs.UploadTimestamp, + LastModified: attrs.LastModified, + Misc: attrs.Info, + Size: attrs.Size, + } + + switch attrs.Status { + case b2.Uploaded: + metadata.Misc["b2-status"] = "Uploaded" + case b2.Folder: + metadata.Misc["b2-status"] = "Folder" + case b2.Hider: + metadata.Misc["b2-status"] = "Hider" + case b2.Started: + metadata.Misc["b2-status"] = "Started" + default: + metadata.Misc["b2-status"] = "Unknown" + } + + return &metadata, nil +} + +func (c *B2InputClient) ID(path string) string { + return fmt.Sprintf("b2://%s/%s%s", c.bucketName, c.prefix, path) +} + +func (c *B2InputClient) GetReader(path string) (io.ReadCloser, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + return obj.NewReader(context.Background()), nil +} diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go new file mode 100644 index 0000000..dca64db --- /dev/null +++ b/internal/client/input/input_client_interface.go @@ -0,0 +1,32 @@ +package input + +import ( + "io" + "time" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" +) + +type InputClient interface { + Scan() ([]string, error) + ReadMetadata(string) (*MetadataStruct, error) + GetReader(string) (io.ReadCloser, error) + ID(path string) string +} + +type MetadataStruct struct { + Name string + StorageType string + Hash string + ContentType string + FirstCreated time.Time + LastModified time.Time + Size int64 + Misc map[string]string +} + +var NewInputClientMap = map[string]func(cfg *config.InputConfig) (InputClient, error){ + "b2": NewB2InputClient, + "s3": NewS3InputClient, + "local-unix": NewLocalUnixInputClient, +} diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go new file mode 100644 index 0000000..21b08a9 --- /dev/null +++ b/internal/client/input/local_unix.go @@ -0,0 +1,108 @@ +package input + +import ( + "fmt" + "io" + "log/slog" + "mime" + "os" + "slices" + "strconv" + "strings" + "syscall" + "time" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" +) + +var _ InputClient = (*LocalUnixInputClient)(nil) + +type LocalUnixInputClient struct { + path string + maxDepth int + knownExtensions []string +} + +func NewLocalUnixInputClient(cfg *config.InputConfig) (InputClient, error) { + if cfg.Storage.Type != "local-unix" { + return nil, fmt.Errorf("invalid storage type for LocalUnixInputClient") + } + localCfg := cfg.Storage.Config.(*config.InputLocalUnixConfig) + + return &LocalUnixInputClient{ + path: localCfg.Path, + maxDepth: localCfg.MaxDepth, + knownExtensions: cfg.KnownExtensions, + }, nil +} + +func (c *LocalUnixInputClient) Scan() ([]string, error) { + return c.recursiveScan(c.path, c.maxDepth) +} + +func (c *LocalUnixInputClient) recursiveScan(dir string, depth int) ([]string, error) { + filePaths := make([]string, 0) + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("fail to read directory: %w", err) + } + + for _, entry := range entries { + if entry.IsDir() && depth > 0 { + subFilePaths, err := c.recursiveScan(dir+entry.Name()+"/", depth-1) + if err != nil { + return nil, fmt.Errorf("fail to scan subdirectory '%s': %w", entry.Name(), err) + } + filePaths = append(filePaths, subFilePaths...) + } else if !entry.IsDir() { + nameParts := strings.Split(entry.Name(), ".") + if len(nameParts) < 2 { + continue + } + if slices.Contains(c.knownExtensions, strings.ToLower(nameParts[len(nameParts)-1])) { + filePaths = append(filePaths, strings.TrimPrefix(dir+entry.Name(), c.path)) + } + fmt.Println(entry.Name()) + } + } + + return filePaths, nil +} + +func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error) { + nodePathParts := strings.Split(path, "/") + nodeName := nodePathParts[len(nodePathParts)-1] + nodeNameParts := strings.Split(nodeName, ".") + nodeExt := "" + if len(nodeNameParts) >= 2 { + nodeExt = nodeNameParts[len(nodeNameParts)-1] + } + slog.Debug("got a file extension", slog.String("extension", nodeExt), slog.String("path", path), slog.String("filename", nodeName)) + + fileInfo, err := os.Stat(c.path + path) + if err != nil { + return nil, fmt.Errorf("fail to read file info: %w", err) + } + + stat_t := fileInfo.Sys().(*syscall.Stat_t) + creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec) + + return &MetadataStruct{ + Name: fileInfo.Name(), + StorageType: "local-unix", + Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16), + ContentType: mime.TypeByExtension("." + nodeExt), + FirstCreated: creationTime, + LastModified: fileInfo.ModTime(), + Size: fileInfo.Size(), + Misc: map[string]string{}, + }, nil +} + +func (c *LocalUnixInputClient) ID(path string) string { + return fmt.Sprintf("local-unix://%s%s", c.path, path) +} + +func (c *LocalUnixInputClient) GetReader(path string) (io.ReadCloser, error) { + return os.Open(c.path + path) +} diff --git a/internal/client/input/s3.go b/internal/client/input/s3.go new file mode 100644 index 0000000..00684c2 --- /dev/null +++ b/internal/client/input/s3.go @@ -0,0 +1,157 @@ +package input + +import ( + "context" + "fmt" + "io" + "slices" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" +) + +var _ InputClient = (*S3InputClient)(nil) + +type S3InputClient struct { + prefix string + bucketName string + s3cl *s3.Client + knownExtensions []string +} + +func NewS3InputClient(cfg *config.InputConfig) (InputClient, error) { + if cfg.Storage.Type != "s3" { + return nil, fmt.Errorf("invalid storage type for S3InputClient") + } + s3cfg := cfg.Storage.Config.(*config.S3Config) + + s3cl, err := newS3Client(s3cfg) + if err != nil { + return nil, fmt.Errorf("create S3 client: %w", err) + } + + return &S3InputClient{ + s3cl: s3cl, + bucketName: s3cfg.BucketName, + prefix: s3cfg.Prefix, + knownExtensions: cfg.KnownExtensions, + }, nil +} + +func (c *S3InputClient) Scan() ([]string, error) { + var filePaths []string + + paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucketName), + Prefix: aws.String(c.prefix), + }) + + for paginator.HasMorePages() { + output, err := paginator.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("list S3 objects: %w", err) + } + + for _, obj := range output.Contents { + name := aws.ToString(obj.Key) + + if len(c.knownExtensions) != 0 { + nameParts := strings.Split(name, ".") + if len(nameParts) < 2 { + continue + } + ext := strings.ToLower(nameParts[len(nameParts)-1]) + if !slices.Contains(c.knownExtensions, ext) { + continue + } + } + + filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix)) + } + } + + return filePaths, nil +} + +func (c *S3InputClient) ReadMetadata(path string) (*MetadataStruct, error) { + key := c.prefix + path + + head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("head S3 object %s: %w", key, err) + } + + metadata := MetadataStruct{ + Name: key, + StorageType: "s3", + Hash: strings.Trim(aws.ToString(head.ETag), "\""), + ContentType: aws.ToString(head.ContentType), + Misc: head.Metadata, + } + + if head.ContentLength != nil { + metadata.Size = *head.ContentLength + } + if head.LastModified != nil { + metadata.LastModified = *head.LastModified + metadata.FirstCreated = *head.LastModified + } + + return &metadata, nil +} + +func (c *S3InputClient) ID(path string) string { + return fmt.Sprintf("s3://%s/%s%s", c.bucketName, c.prefix, path) +} + +func (c *S3InputClient) GetReader(path string) (io.ReadCloser, error) { + key := c.prefix + path + + output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("get S3 object %s: %w", key, err) + } + + return output.Body, nil +} + +func newS3Client(cfg *config.S3Config) (*s3.Client, error) { + opts := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(cfg.Region), + } + + if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" { + opts = append(opts, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""), + )) + } + + awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, fmt.Errorf("load AWS config: %w", err) + } + + var s3Opts []func(*s3.Options) + if cfg.Endpoint != "" { + s3Opts = append(s3Opts, func(o *s3.Options) { + o.BaseEndpoint = aws.String(cfg.Endpoint) + }) + } + s3Opts = append(s3Opts, func(o *s3.Options) { + o.UsePathStyle = cfg.UsePathStyle + o.DisableLogOutputChecksumValidationSkipped = true + }) + + return s3.NewFromConfig(awsCfg, s3Opts...), nil +} diff --git a/internal/client/output/b2.go b/internal/client/output/b2.go new file mode 100644 index 0000000..0616368 --- /dev/null +++ b/internal/client/output/b2.go @@ -0,0 +1,113 @@ +package output + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + + "github.com/Backblaze/blazer/b2" + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" +) + +var _ OutputClient = (*B2OutputClient)(nil) + +type B2OutputClient struct { + prefix string + bucket *b2.Bucket + b2cl *b2.Client +} + +func NewB2OutputClient(cfg *config.OutputConfig) (OutputClient, error) { + if cfg.Storage.Type != "b2" { + return nil, fmt.Errorf("invalid storage type for B2OutputClient") + } + b2cfg := cfg.Storage.Config.(*config.B2Config) + + b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) + if err != nil { + return nil, err + } + + bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) + if err != nil { + return nil, err + } + + return &B2OutputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil +} + +func (c *B2OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs := &b2.Attrs{Info: make(map[string]string)} + attrs.Info["sha1-original"] = inputMetadata.Hash + attrs.ContentType = outputContentType + + return obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs)), nil +} + +func (c *B2OutputClient) ReadMetadata(path string) (*MetadataStruct, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + metadata := MetadataStruct{ + Name: attrs.Name, + StorageType: "b2", + Hash: attrs.SHA1, + HashOriginal: attrs.Info["sha1-original"], + ContentType: attrs.ContentType, + FirstCreated: attrs.UploadTimestamp, + LastModified: attrs.LastModified, + Misc: attrs.Info, + Size: attrs.Size, + } + + switch attrs.Status { + case b2.Uploaded: + metadata.Misc["b2-status"] = "Uploaded" + case b2.Folder: + metadata.Misc["b2-status"] = "Folder" + case b2.Hider: + metadata.Misc["b2-status"] = "Hider" + case b2.Started: + metadata.Misc["b2-status"] = "Started" + default: + metadata.Misc["b2-status"] = "Unknown" + } + + return &metadata, nil +} + +func (c *B2OutputClient) IsMissing(path string) bool { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return true + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return true + } + + attrsJson, _ := json.Marshal(attrs) + slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(attrsJson))) + + if attrs.Size == 0 { + return true + } + + return attrs.Status == b2.Hider +} diff --git a/internal/client/output/local_unix.go b/internal/client/output/local_unix.go new file mode 100644 index 0000000..6f9a7b3 --- /dev/null +++ b/internal/client/output/local_unix.go @@ -0,0 +1,129 @@ +package output + +import ( + "fmt" + "io" + "log/slog" + "mime" + "os" + "strconv" + "strings" + "syscall" + "time" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" + "golang.org/x/sys/unix" +) + +var _ OutputClient = (*LocalUnixOutputClient)(nil) + +type LocalUnixOutputClient struct { + path string + fileMode uint32 + dirMode uint32 + attrMode string +} + +func NewLocalUnixOutputClient(cfg *config.OutputConfig) (OutputClient, error) { + if cfg.Storage.Type != "local-unix" { + return nil, fmt.Errorf("invalid storage type for LocalUnixOutputClient") + } + localCfg := cfg.Storage.Config.(*config.OutputLocalUnixConfig) + + fpm, err := strconv.ParseInt(localCfg.FilePermissionMode, 8, 32) + if err != nil { + return nil, fmt.Errorf("fail to parse file permission mode as an octal number: %w", err) + } + + dpm, err := strconv.ParseInt(localCfg.DirPermissionMode, 8, 32) + if err != nil { + return nil, fmt.Errorf("fail to parse directory permission mode as an octal number: %w", err) + } + + return &LocalUnixOutputClient{localCfg.Path, uint32(fpm), uint32(dpm), localCfg.AttributesImplementation}, nil +} + +func (c *LocalUnixOutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, _ string) (io.WriteCloser, error) { + pathSegments := strings.Split(path, "/") + dirpath := strings.Join(pathSegments[0:len(pathSegments)-1], "/") + if err := os.MkdirAll(c.path+dirpath, os.FileMode(c.dirMode)); err != nil { + return nil, fmt.Errorf("fail to mkdir parent directories for a path: %w", err) + } + + var err error + if _, err = os.Stat(c.path + path); err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("fail to stat a file, although file exists: %w", err) + } + if os.IsNotExist(err) { + f, err := os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE, os.FileMode(c.fileMode)) + if err != nil { + return nil, fmt.Errorf("fail to create a file: %w", err) + } + f.Close() + } + + switch c.attrMode { + case "xattr": + if err := unix.Setxattr(c.path+path, "user.originalfile.mddate", []byte(strconv.FormatInt(inputMetadata.LastModified.Unix(), 16)), 0); err != nil { + return nil, fmt.Errorf("fail to write user.originalfile.mddate xattribute: %w", err) + } + case "none": + default: + return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode) + } + + return os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(c.fileMode)) +} + +func (c *LocalUnixOutputClient) ReadMetadata(path string) (*MetadataStruct, error) { + nodePathParts := strings.Split(path, "/") + nodeName := nodePathParts[len(nodePathParts)-1] + nodeNameParts := strings.Split(nodeName, ".") + nodeExt := "" + if len(nodeNameParts) >= 2 { + nodeExt = nodeNameParts[len(nodeNameParts)-1] + } + + fileInfo, err := os.Stat(c.path + path) + if err != nil { + return nil, fmt.Errorf("fail to read file info: %w", err) + } + + stat_t := fileInfo.Sys().(*syscall.Stat_t) + creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec) + + mddateOriginal := make([]byte, 0) + switch c.attrMode { + case "xattr": + sz, err := unix.Getxattr(c.path+path, "user.originalfile.mddate", nil) + if err != nil { + slog.Warn("fail to get size of user.originalfile.mddate attribute, proceeding as if no such attribute is there", slog.String("error", err.Error())) + break + } + mddateOriginal = make([]byte, sz) + if _, err = unix.Getxattr(c.path+path, "user.originalfile.mddate", mddateOriginal); err != nil { + return nil, fmt.Errorf("fail to get user.originalfile.mddate attribute: %w", err) + } + case "none": + default: + return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode) + } + + return &MetadataStruct{ + Name: fileInfo.Name(), + StorageType: "local-unix", + Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16), + HashOriginal: string(mddateOriginal), + ContentType: mime.TypeByExtension("." + nodeExt), + FirstCreated: creationTime, + LastModified: fileInfo.ModTime(), + Size: fileInfo.Size(), + Misc: map[string]string{}, + }, nil +} + +func (c *LocalUnixOutputClient) IsMissing(path string) bool { + _, err := os.Stat(c.path + path) + return err != nil +} diff --git a/internal/client/output/output_client_interface.go b/internal/client/output/output_client_interface.go new file mode 100644 index 0000000..4415aa4 --- /dev/null +++ b/internal/client/output/output_client_interface.go @@ -0,0 +1,33 @@ +package output + +import ( + "io" + "time" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" +) + +type OutputClient interface { + GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) + ReadMetadata(path string) (*MetadataStruct, error) + IsMissing(path string) bool +} + +type MetadataStruct struct { + Name string + StorageType string + Hash string + HashOriginal string + ContentType string + FirstCreated time.Time + LastModified time.Time + Size int64 + Misc map[string]string +} + +var NewOutputClientMap = map[string]func(cfg *config.OutputConfig) (OutputClient, error){ + "b2": NewB2OutputClient, + "s3": NewS3OutputClient, + "local-unix": NewLocalUnixOutputClient, +} diff --git a/internal/client/output/s3.go b/internal/client/output/s3.go new file mode 100644 index 0000000..f0527fb --- /dev/null +++ b/internal/client/output/s3.go @@ -0,0 +1,170 @@ +package output + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" +) + +var _ OutputClient = (*S3OutputClient)(nil) + +type S3OutputClient struct { + prefix string + bucketName string + s3cl *s3.Client +} + +func NewS3OutputClient(cfg *config.OutputConfig) (OutputClient, error) { + if cfg.Storage.Type != "s3" { + return nil, fmt.Errorf("invalid storage type for S3OutputClient") + } + s3cfg := cfg.Storage.Config.(*config.S3Config) + + s3cl, err := newS3Client(s3cfg) + if err != nil { + return nil, fmt.Errorf("create S3 client: %w", err) + } + + return &S3OutputClient{ + s3cl: s3cl, + bucketName: s3cfg.BucketName, + prefix: s3cfg.Prefix, + }, nil +} + +func (c *S3OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) { + key := c.prefix + path + + return &s3WriteCloser{ + key: key, + bucketName: c.bucketName, + s3cl: c.s3cl, + contentType: outputContentType, + hashOriginal: inputMetadata.Hash, + buf: &bytes.Buffer{}, + }, nil +} + +func (c *S3OutputClient) ReadMetadata(path string) (*MetadataStruct, error) { + key := c.prefix + path + + head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("head S3 object %s: %w", key, err) + } + + metadata := MetadataStruct{ + Name: key, + StorageType: "s3", + Hash: strings.Trim(aws.ToString(head.ETag), "\""), + HashOriginal: head.Metadata["sha1-original"], + ContentType: aws.ToString(head.ContentType), + Misc: head.Metadata, + } + + if head.ContentLength != nil { + metadata.Size = *head.ContentLength + } + if head.LastModified != nil { + metadata.LastModified = *head.LastModified + metadata.FirstCreated = *head.LastModified + } + + return &metadata, nil +} + +func (c *S3OutputClient) IsMissing(path string) bool { + key := c.prefix + path + + head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(c.bucketName), + Key: aws.String(key), + }) + if err != nil { + return true + } + + headJson, _ := json.Marshal(head) + slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(headJson))) + + if head.ContentLength == nil || *head.ContentLength == 0 { + return true + } + + return false +} + +// s3WriteCloser buffers writes and uploads to S3 on Close. +type s3WriteCloser struct { + key string + bucketName string + s3cl *s3.Client + contentType string + hashOriginal string + buf *bytes.Buffer +} + +func (w *s3WriteCloser) Write(p []byte) (int, error) { + return w.buf.Write(p) +} + +func (w *s3WriteCloser) Close() error { + _, err := w.s3cl.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(w.bucketName), + Key: aws.String(w.key), + Body: bytes.NewReader(w.buf.Bytes()), + ContentType: aws.String(w.contentType), + Metadata: map[string]string{ + "sha1-original": w.hashOriginal, + }, + }) + if err != nil { + return fmt.Errorf("put S3 object %s: %w", w.key, err) + } + return nil +} + +func newS3Client(cfg *config.S3Config) (*s3.Client, error) { + opts := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(cfg.Region), + } + + if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" { + opts = append(opts, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""), + )) + } + + awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, fmt.Errorf("load AWS config: %w", err) + } + + var s3Opts []func(*s3.Options) + if cfg.Endpoint != "" { + s3Opts = append(s3Opts, func(o *s3.Options) { + o.BaseEndpoint = aws.String(cfg.Endpoint) + }) + } + s3Opts = append(s3Opts, func(o *s3.Options) { + o.UsePathStyle = cfg.UsePathStyle + o.DisableLogOutputChecksumValidationSkipped = true + }) + + return s3.NewFromConfig(awsCfg, s3Opts...), nil +} diff --git a/internal/converter/converter_interface.go b/internal/converter/converter_interface.go new file mode 100644 index 0000000..abff99d --- /dev/null +++ b/internal/converter/converter_interface.go @@ -0,0 +1,21 @@ +package converter + +import ( + "io" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" +) + +type Converter interface { + Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error + DeductOutputPath(inputPath string) string + ReadMetadata(path string) (*output.MetadataStruct, error) + IsMissing(path string) bool +} + +var NewConverterMap = map[string]func(cfg *config.ConverterConfig) (Converter, error){ + "webp": NewWebpConverter, + "jpeg": NewJpegConverter, +} diff --git a/internal/converter/jpeg.go b/internal/converter/jpeg.go new file mode 100644 index 0000000..a35a47c --- /dev/null +++ b/internal/converter/jpeg.go @@ -0,0 +1,114 @@ +package converter + +import ( + "fmt" + "image" + "image/jpeg" + "image/png" + "io" + "log/slog" + "path/filepath" + "strings" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" + "github.com/kolesa-team/go-webp/webp" + "golang.org/x/image/draw" +) + +var _ Converter = (*JpegConverter)(nil) + +type JpegConverter struct { + maxWidth int + maxHeight int + extensionName string + quality int + outputClient output.OutputClient +} + +func NewJpegConverter(cfg *config.ConverterConfig) (Converter, error) { + if cfg.Type != "jpeg" { + return nil, fmt.Errorf("invalid storage type for JpegConverter") + } + jpegCfg := cfg.Config.(*config.JpegConfig) + + outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output) + if err != nil { + return nil, fmt.Errorf("fail to initialize output client: %w", err) + } + + extensionName := ".jpg" + if jpegCfg.ExtensionName != "" { + extensionName = "." + jpegCfg.ExtensionName + } + + return &JpegConverter{jpegCfg.Size.MaxWidth, jpegCfg.Size.MaxHeight, extensionName, jpegCfg.Quality, outputClient}, nil +} + +func (p *JpegConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error { + var src image.Image + + writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/jpeg") + if err != nil { + return fmt.Errorf("fail to initialize writer for output: %w", err) + } + defer writer.Close() + + switch inputMetadata.ContentType { + case "image/jpeg": + src, err = jpeg.Decode(reader) + if err != nil { + return fmt.Errorf("decode jpeg: %w", err) + } + case "image/png": + src, err = png.Decode(reader) + if err != nil { + return fmt.Errorf("decode png: %w", err) + } + case "image/webp": + src, err = webp.Decode(reader, nil) + if err != nil { + return fmt.Errorf("decode webp: %w", err) + } + default: + return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType) + } + + xCoef := 1.0 + if p.maxWidth > 0 { + xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X) + } + yCoef := 1.0 + if p.maxHeight > 0 { + yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y) + } + slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef)) + + minCoef := xCoef + if yCoef < minCoef { + minCoef = yCoef + } + + if minCoef >= 1.0 { + return jpeg.Encode(writer, src, &jpeg.Options{Quality: p.quality}) + } + + dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5))) + draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil) + + return jpeg.Encode(writer, dst, &jpeg.Options{Quality: p.quality}) +} + +func (p *JpegConverter) DeductOutputPath(inputPath string) string { + withoutExt, _ := strings.CutSuffix(inputPath, filepath.Ext(inputPath)) + return withoutExt + p.extensionName +} + +func (p *JpegConverter) ReadMetadata(path string) (*output.MetadataStruct, error) { + return p.outputClient.ReadMetadata(path) +} + +func (p *JpegConverter) IsMissing(path string) bool { + return p.outputClient.IsMissing(path) +} diff --git a/internal/converter/webp.go b/internal/converter/webp.go new file mode 100644 index 0000000..fbffd01 --- /dev/null +++ b/internal/converter/webp.go @@ -0,0 +1,118 @@ +package converter + +import ( + "fmt" + "image" + "image/jpeg" + "image/png" + "io" + "log/slog" + "strings" + + "golang.org/x/image/draw" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" + "github.com/kolesa-team/go-webp/encoder" + "github.com/kolesa-team/go-webp/webp" +) + +var _ Converter = (*WebpConverter)(nil) + +type WebpConverter struct { + maxWidth int + maxHeight int + quality int + outputClient output.OutputClient +} + +func NewWebpConverter(cfg *config.ConverterConfig) (Converter, error) { + if cfg.Type != "webp" { + return nil, fmt.Errorf("invalid storage type for WebpConverter") + } + webpCfg := cfg.Config.(*config.WebpConfig) + + outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output) + if err != nil { + return nil, fmt.Errorf("fail to initialize output client: %w", err) + } + + return &WebpConverter{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality, outputClient}, nil +} + +func (p *WebpConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error { + var src image.Image + + writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/webp") + if err != nil { + return fmt.Errorf("fail to initialize writer for output: %w", err) + } + defer writer.Close() + + switch inputMetadata.ContentType { + case "image/jpeg": + src, err = jpeg.Decode(reader) + if err != nil { + return fmt.Errorf("decode jpeg: %w", err) + } + case "image/png": + src, err = png.Decode(reader) + if err != nil { + return fmt.Errorf("decode png: %w", err) + } + case "image/webp": + src, err = webp.Decode(reader, nil) + if err != nil { + return fmt.Errorf("decode webp: %w", err) + } + default: + return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType) + } + + opts, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, float32(p.quality)) + if err != nil { + return fmt.Errorf("create webp encoder options: %w", err) + } + + xCoef := 1.0 + if p.maxWidth > 0 { + xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X) + } + yCoef := 1.0 + if p.maxHeight > 0 { + yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y) + } + slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef)) + + minCoef := xCoef + if yCoef < minCoef { + minCoef = yCoef + } + + if minCoef >= 1.0 { + return webp.Encode(writer, src, opts) + } + + dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5))) + draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil) + + return webp.Encode(writer, dst, opts) +} + +func (p *WebpConverter) DeductOutputPath(inputPath string) string { + pathParts := strings.Split(inputPath, ".") + if len(pathParts) < 2 { + return inputPath + ".webp" + } + pathParts[len(pathParts)-1] = "webp" + return strings.Join(pathParts, ".") +} + +func (p *WebpConverter) ReadMetadata(path string) (*output.MetadataStruct, error) { + return p.outputClient.ReadMetadata(path) +} + +func (p *WebpConverter) IsMissing(path string) bool { + return p.outputClient.IsMissing(path) +} @@ -1,20 +1,302 @@ package main import ( + "bytes" + "encoding/csv" + "encoding/json" "flag" + "hash/crc32" + "io" + "log/slog" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" + "github.com/SayaAndy/saya-today-thumbnail-generator/internal/converter" ) var ( - configPath = flag.String("c", "config.json", "Path to the configuration file") + configPath = flag.String("c", "config.json", "Path to the configuration file") + sigTermChan = make(chan os.Signal, 1) + cacheMapMutex = &sync.RWMutex{} + cacheMap = make(map[string]map[uint32]struct{}) ) func main() { + signal.Notify(sigTermChan, os.Interrupt, syscall.SIGTERM) + flag.Parse() cfg := &config.Config{} if err := config.LoadConfig(*configPath, cfg); err != nil { - panic(err) + slog.Error("fail to load configuration", slog.String("error", err.Error())) + os.Exit(1) + } + + slog.SetLogLoggerLevel(cfg.LogLevel) + slog.Info("starting thumbnail generator...") + + select { + case <-sigTermChan: + slog.Info("exiting due to termination signal") + os.Exit(130) + default: + } + + inputClient, err := input.NewInputClientMap[cfg.Input.Storage.Type](&cfg.Input) + if err != nil { + slog.Error("fail to initialize input client", slog.String("error", err.Error())) + os.Exit(1) + } + + converters := make([]converter.Converter, 0, len(cfg.Converters)) + converterTypes := make([]string, 0, len(cfg.Converters)) + converterHashes := make([]uint32, 0, len(cfg.Converters)) + for _, converterCfg := range cfg.Converters { + converterBytes, _ := json.Marshal(converterCfg) + conv, err := converter.NewConverterMap[converterCfg.Type](&converterCfg) + if err != nil { + slog.Error("fail to initialize converter", slog.String("error", err.Error())) + os.Exit(1) + } + converters = append(converters, conv) + converterTypes = append(converterTypes, converterCfg.Type) + converterHashes = append(converterHashes, crc32.ChecksumIEEE(converterBytes)) + } + + generalLogger := slog.With(slog.String("input_storage", cfg.Input.Storage.Type)) + generalLogger.Info("initialized input client and converters", slog.String("converter_types", strings.Join(converterTypes, " "))) + + select { + case <-sigTermChan: + generalLogger.Info("exiting due to termination signal") + os.Exit(130) + default: + } + + files, err := inputClient.Scan() + if err != nil { + generalLogger.Error("fail to scan input files", slog.String("error", err.Error())) + os.Exit(1) + } + fileCount := len(files) + generalLogger.Info("scanned files", slog.Int("file_count", fileCount)) + + select { + case <-sigTermChan: + generalLogger.Info("exiting due to termination signal") + os.Exit(130) + default: + } + + if cfg.Input.CacheProcessed { + cacheFile, err := os.OpenFile(cfg.Input.CacheProcessedCsvPath, os.O_CREATE|os.O_RDONLY, 0644) + if err != nil { + generalLogger.Error("fail to initialize cache file", slog.String("cache_path", cfg.Input.CacheProcessedCsvPath), slog.String("error", err.Error())) + os.Exit(1) + } + defer cacheFile.Close() + + csvReader := csv.NewReader(cacheFile) + for { + rec, err := csvReader.Read() + if err == io.EOF { + break + } + if err != nil { + generalLogger.Error("fail to initialize cache while reading file", slog.String("cache_path", cfg.Input.CacheProcessedCsvPath), slog.String("error", err.Error())) + os.Exit(1) + } + if len(rec) != 2 { + generalLogger.Error("fail to initialize cache while reading file", slog.Int("record_length", len(rec)), slog.String("error", "incorrect format: expected '{image-name},{semicolon-separated-processor-hashes}'")) + os.Exit(1) + } + hashes := strings.Split(rec[1], ";") + cacheMap[rec[0]] = make(map[uint32]struct{}) + for _, hash := range hashes { + hashUint, err := strconv.ParseUint(hash, 10, 32) + if err != nil { + generalLogger.Error("fail to initialize cache while reading file", slog.Int("record_length", len(rec)), slog.String("error", err.Error())) + } + cacheMap[rec[0]][uint32(hashUint)] = struct{}{} + } + } + } + + select { + case <-sigTermChan: + generalLogger.Info("exiting due to termination signal") + os.Exit(130) + default: + } + + processSemaphore := make(chan struct{}, cfg.MaxProcessThreads) + queueSemaphore := make(chan struct{}, cfg.MaxPreProcessThreads) + var wg sync.WaitGroup + wg.Add(fileCount) + + processTerminating := false + + for i, file := range files { + queueSemaphore <- struct{}{} + + select { + case <-sigTermChan: + generalLogger.Info("exiting due to termination signal") + processTerminating = true + default: + } + + go func(index int, inputName string, earlyTerminate bool) { + defer func() { <-queueSemaphore; wg.Done() }() + + fileLogger := generalLogger.With(slog.String("input_path", inputName), slog.Int("file_index", index)) + + if earlyTerminate { + fileLogger.Info("skip processing file (process is terminating)") + return + } + + var inputMetadata *input.MetadataStruct + + id := inputClient.ID(file) + cacheMapMutex.Lock() + if _, ok := cacheMap[id]; !ok { + cacheMap[id] = make(map[uint32]struct{}) + } + cacheMapMutex.Unlock() + + convertersToLaunch := []int{} + for j, conv := range converters { + if cfg.Input.CacheProcessed { + cacheMapMutex.RLock() + if _, ok := cacheMap[id][converterHashes[j]]; ok { + cacheMapMutex.RUnlock() + fileLogger.Info("skip already processed file (based on cache file containing it and processor)", + slog.String("file_id", id), + slog.Uint64("conv_hash", uint64(converterHashes[j])), + slog.Int("conv_index", j)) + continue + } + cacheMapMutex.RUnlock() + } + outputName := conv.DeductOutputPath(inputName) + originalInputHash := "" + convLogger := fileLogger.With(slog.String("output_path", outputName), slog.Int("conv_index", j)) + + if inputMetadata == nil { + inputMetadata, err = inputClient.ReadMetadata(inputName) + if err != nil { + fileLogger.Warn("fail to read metadata of (supposedly existing) input file", slog.String("error", err.Error())) + return + } + } + + switch cfg.Converters[j].Output.RewriteOn { + case "Never": + if !conv.IsMissing(outputName) { + convLogger.Info("skip already existing file") + continue + } + case "UnequalHashInCache": + if !conv.IsMissing(outputName) { + outputMetadata, err := conv.ReadMetadata(outputName) + if err != nil { + convLogger.Warn("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error())) + continue + } + originalInputHash = outputMetadata.HashOriginal + if inputMetadata.Hash == originalInputHash { + convLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash)) + cacheMapMutex.Lock() + cacheMap[id][converterHashes[j]] = struct{}{} + cacheMapMutex.Unlock() + continue + } + } + case "Always": + } + + convertersToLaunch = append(convertersToLaunch, j) + } + + if len(convertersToLaunch) == 0 { + return + } + + processSemaphore <- struct{}{} + defer func() { <-processSemaphore }() + + fileLogger.Info("start to process file", slog.String("input_hash", inputMetadata.Hash)) + reader, err := inputClient.GetReader(inputName) + if err != nil { + fileLogger.Warn("fail to get reader for input file", slog.String("error", err.Error())) + return + } + + fileContent, err := io.ReadAll(reader) + if err != nil { + fileLogger.Warn("fail to read content of input file", slog.String("error", err.Error())) + return + } else if int64(len(fileContent)) != inputMetadata.Size { + fileLogger.Warn("the downloaded file seems to be missing some of its content", + slog.Int("actual_size_bytes", len(fileContent)), + slog.Int64("expected_size_bytes", inputMetadata.Size)) + } + reader.Close() + + for _, convIndex := range convertersToLaunch { + conv := converters[convIndex] + outputName := conv.DeductOutputPath(inputName) + convLogger := fileLogger.With(slog.String("output_path", outputName), slog.Int("conv_index", convIndex)) + + if err := conv.Process(inputMetadata, bytes.NewReader(fileContent), outputName); err != nil { + convLogger.Warn("fail to convert file", slog.String("error", err.Error())) + return + } + + convLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash)) + cacheMapMutex.Lock() + cacheMap[id][converterHashes[convIndex]] = struct{}{} + cacheMapMutex.Unlock() + } + }(i, file, processTerminating) + } + + wg.Wait() + + select { + case <-sigTermChan: + generalLogger.Info("exiting due to termination signal") + os.Exit(130) + default: + } + + generalLogger.Info("all files processed successfully") + + if cfg.Input.CacheProcessed { + generalLogger.Info("writing cache file") + cacheFile, err := os.OpenFile(cfg.Input.CacheProcessedCsvPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + generalLogger.Error("error writing cache into file") + } else { + for id, hashes := range cacheMap { + var hashStrings = make([]string, 0, len(hashes)) + for hash := range hashes { + hashStrings = append(hashStrings, strconv.FormatUint(uint64(hash), 10)) + } + cacheFile.Write([]byte(id)) + cacheFile.Write([]byte{','}) + cacheFile.WriteString(strings.Join(hashStrings, ";")) + cacheFile.Write([]byte{'\n'}) + } + } } + + os.Exit(0) } |