summaryrefslogtreecommitdiff
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore10
-rw-r--r--CLAUDE.md42
-rw-r--r--config/config-b2.sample.json141
-rw-r--r--config/config-local-unix.sample.json41
-rw-r--r--config/config-s3.sample.json53
-rw-r--r--config/config.go172
-rw-r--r--config/config.json31
-rw-r--r--go.mod39
-rw-r--r--go.sum73
-rw-r--r--internal/client/input/b2.go135
-rw-r--r--internal/client/input/input_client_interface.go32
-rw-r--r--internal/client/input/local_unix.go108
-rw-r--r--internal/client/input/s3.go157
-rw-r--r--internal/client/output/b2.go113
-rw-r--r--internal/client/output/local_unix.go129
-rw-r--r--internal/client/output/output_client_interface.go33
-rw-r--r--internal/client/output/s3.go170
-rw-r--r--internal/converter/converter_interface.go21
-rw-r--r--internal/converter/jpeg.go114
-rw-r--r--internal/converter/webp.go118
-rw-r--r--main.go286
21 files changed, 76 insertions, 1942 deletions
diff --git a/.gitignore b/.gitignore
index 186750e..aaadf73 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,13 +26,7 @@ go.work.sum
# env file
.env
-.envrc
# Editor/IDE
-.idea/
-.vscode/
-
-config*.json
-!config*.sample.json
-
-cache.csv
+# .idea/
+# .vscode/
diff --git a/CLAUDE.md b/CLAUDE.md
deleted file mode 100644
index 94be9b2..0000000
--- a/CLAUDE.md
+++ /dev/null
@@ -1,42 +0,0 @@
-# CLAUDE.md
-
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
-
-## Build & Run
-
-```bash
-go build -o saya-today-thumbnail-generator
-./saya-today-thumbnail-generator -c config/config-local-unix.sample.json
-```
-
-No Makefile. No test files exist yet — `go test ./...` would be the convention.
-
-## What This Is
-
-Concurrent batch thumbnail generator. Reads images from a source (Backblaze B2, S3/S3-compatible, or local filesystem), converts to WebP/JPEG with configurable quality/size, writes to output storage. Maintains a CSV-based file cache to skip already-processed images.
-
-## Architecture
-
-Three pluggable abstractions, each using a factory/registry pattern keyed by string type:
-
-- **InputClient** (`internal/client/input/`) — scans source files, provides readers. Implementations: `b2`, `s3`, `local-unix`
-- **OutputClient** (`internal/client/output/`) — writes converted files. Implementations: `b2`, `s3`, `local-unix`
-- **Converter** (`internal/converter/`) — decodes image, resizes (Catmull-Rom), encodes to target format. Implementations: `webp`, `jpeg`
-
-Each has a `New*Map` factory function returning a map of type-string → constructor.
-
-## Config System (`config/config.go`)
-
-JSON config with discriminated unions — `"Type"` field selects which struct to unmarshal into. Supports `${ENV_VAR}` expansion in string values. Validated with go-playground/validator.
-
-Key config knobs: `MaxProcessThreads` (conversion concurrency), `MaxPreProcessThreads` (I/O concurrency), `RewriteOn` strategy per converter (`Never`/`UnequalHashInCache`/`Always`).
-
-Sample configs in `config/config-*.sample.json`. Actual configs are gitignored.
-
-## Concurrency Model
-
-Two-tier semaphore system in `main.go`: pre-process (file scanning/reading) and process (image conversion) run with separate thread pool limits. Graceful shutdown on SIGTERM/SIGINT via early-termination flag. Cache map protected by `sync.RWMutex`.
-
-## Environment
-
-Uses direnv (`.envrc`) for credentials (`B2_KEY_ID`, `B2_APPLICATION_KEY`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`). S3 client supports custom endpoints and path-style addressing for S3-compatible stores (MinIO, etc.).
diff --git a/config/config-b2.sample.json b/config/config-b2.sample.json
deleted file mode 100644
index 60022dd..0000000
--- a/config/config-b2.sample.json
+++ /dev/null
@@ -1,141 +0,0 @@
-{
- "MaxProcessThreads": 4,
- "MaxPreProcessThreads": 12,
- "LogLevel": "info",
- "Input": {
- "Storage": {
- "Type": "b2",
- "Config": {
- "BucketName": "sayana-photos",
- "Region": "eu-central-003",
- "Prefix": "full/",
- "KeyID": "${B2_KEY_ID}",
- "ApplicationKey": "${B2_APPLICATION_KEY}"
- }
- },
- "KnownExtensions": [
- "jpg",
- "jpeg",
- "png"
- ],
- "CacheProcessed": true,
- "CacheProcessedCsvPath": "cache.csv"
- },
- "Converters": [
- {
- "Type": "webp",
- "Config": {
- "Quality": 80,
- "Size": {
- "MaxWidth": 320,
- "MaxHeight": 0
- }
- },
- "Output": {
- "RewriteOn": "UnequalHashInCache",
- "Storage": {
- "Type": "b2",
- "Config": {
- "BucketName": "sayana-photos",
- "Region": "eu-central-003",
- "Prefix": "webp-320p/",
- "KeyID": "${B2_KEY_ID}",
- "ApplicationKey": "${B2_APPLICATION_KEY}"
- }
- }
- }
- },
- {
- "Type": "webp",
- "Config": {
- "Quality": 80,
- "Size": {
- "MaxWidth": 560,
- "MaxHeight": 0
- }
- },
- "Output": {
- "RewriteOn": "UnequalHashInCache",
- "Storage": {
- "Type": "b2",
- "Config": {
- "BucketName": "sayana-photos",
- "Region": "eu-central-003",
- "Prefix": "webp-560p/",
- "KeyID": "${B2_KEY_ID}",
- "ApplicationKey": "${B2_APPLICATION_KEY}"
- }
- }
- }
- },
- {
- "Type": "webp",
- "Config": {
- "Quality": 80,
- "Size": {
- "MaxWidth": 800,
- "MaxHeight": 0
- }
- },
- "Output": {
- "RewriteOn": "UnequalHashInCache",
- "Storage": {
- "Type": "b2",
- "Config": {
- "BucketName": "sayana-photos",
- "Region": "eu-central-003",
- "Prefix": "webp-800p/",
- "KeyID": "${B2_KEY_ID}",
- "ApplicationKey": "${B2_APPLICATION_KEY}"
- }
- }
- }
- },
- {
- "Type": "webp",
- "Config": {
- "Quality": 80,
- "Size": {
- "MaxWidth": 1200,
- "MaxHeight": 0
- }
- },
- "Output": {
- "RewriteOn": "UnequalHashInCache",
- "Storage": {
- "Type": "b2",
- "Config": {
- "BucketName": "sayana-photos",
- "Region": "eu-central-003",
- "Prefix": "webp-1200p/",
- "KeyID": "${B2_KEY_ID}",
- "ApplicationKey": "${B2_APPLICATION_KEY}"
- }
- }
- }
- },
- {
- "Type": "webp",
- "Config": {
- "Quality": 80,
- "Size": {
- "MaxWidth": 1600,
- "MaxHeight": 0
- }
- },
- "Output": {
- "RewriteOn": "UnequalHashInCache",
- "Storage": {
- "Type": "b2",
- "Config": {
- "BucketName": "sayana-photos",
- "Region": "eu-central-003",
- "Prefix": "webp-1600p/",
- "KeyID": "${B2_KEY_ID}",
- "ApplicationKey": "${B2_APPLICATION_KEY}"
- }
- }
- }
- }
- ]
-} \ No newline at end of file
diff --git a/config/config-local-unix.sample.json b/config/config-local-unix.sample.json
deleted file mode 100644
index 941d995..0000000
--- a/config/config-local-unix.sample.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
- "MaxProcessThreads": 4,
- "MaxPreProcessThreads": 12,
- "LogLevel": "debug",
- "Input": {
- "Storage": {
- "Type": "local-unix",
- "Config": {
- "MaxDepth": 3,
- "Path": "/tmp/thumbnailing/full/"
- }
- },
- "KnownExtensions": [
- "jpg",
- "jpeg",
- "png"
- ]
- },
- "Converter": {
- "Type": "webp",
- "Config": {
- "Quality": 80,
- "Size": {
- "MaxWidth": 800,
- "MaxHeight": 0
- }
- },
- "Output": {
- "RewriteOn": "UnequalHashInCache",
- "Storage": {
- "Type": "local-unix",
- "Config": {
- "Path": "/tmp/thumbnailing/thumbnails/",
- "DirPermissionMode": "0755",
- "FilePermissionMode": "0644",
- "AttributesImplementation": "xattr"
- }
- }
- }
- }
-} \ No newline at end of file
diff --git a/config/config-s3.sample.json b/config/config-s3.sample.json
deleted file mode 100644
index 5b067a5..0000000
--- a/config/config-s3.sample.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
- "MaxProcessThreads": 4,
- "MaxPreProcessThreads": 12,
- "LogLevel": "info",
- "Input": {
- "Storage": {
- "Type": "s3",
- "Config": {
- "BucketName": "my-photos",
- "Region": "",
- "Prefix": "full/",
- "Endpoint": "https://minio.local",
- "UsePathStyle": true,
- "AccessKeyID": "${AWS_ACCESS_KEY_ID}",
- "SecretAccessKey": "${AWS_SECRET_ACCESS_KEY}"
- }
- },
- "KnownExtensions": [
- "jpg",
- "jpeg",
- "png"
- ],
- "CacheProcessed": true,
- "CacheProcessedCsvPath": "cache.csv"
- },
- "Converters": [
- {
- "Type": "webp",
- "Config": {
- "Quality": 80,
- "Size": {
- "MaxWidth": 320,
- "MaxHeight": 0
- }
- },
- "Output": {
- "RewriteOn": "UnequalHashInCache",
- "Storage": {
- "Type": "s3",
- "Config": {
- "BucketName": "my-photos",
- "Region": "",
- "Prefix": "webp-320p/",
- "Endpoint": "https://minio.local",
- "UsePathStyle": true,
- "AccessKeyID": "${AWS_ACCESS_KEY_ID}",
- "SecretAccessKey": "${AWS_SECRET_ACCESS_KEY}"
- }
- }
- }
- }
- ]
-}
diff --git a/config/config.go b/config/config.go
index 06fa94f..6d37904 100644
--- a/config/config.go
+++ b/config/config.go
@@ -3,132 +3,32 @@ package config
import (
"encoding/json"
"fmt"
- "log/slog"
"os"
"github.com/go-playground/validator/v10"
)
type Config struct {
- Input InputConfig `json:"Input" validate:"required"`
- Converters []ConverterConfig `json:"Converters" validate:"required"`
- MaxProcessThreads int `json:"MaxProcessThreads" validate:"required,min=1"`
- MaxPreProcessThreads int `json:"MaxPreProcessThreads" validate:"min=1;gtefield=MaxProcessThreads"`
- LogLevel slog.Level `json:"LogLevel" validate:"required"`
-}
-
-type InputConfig struct {
- Storage InputStorageConfig `json:"Storage" validate:"required"`
- KnownExtensions []string `json:"KnownExtensions" validate:"required,min=0,dive,min=1"`
- CacheProcessed bool `json:"CacheProcessed"`
- CacheProcessedCsvPath string `json:"CacheProcessedCsvPath" validate:"filepath"`
-}
-
-type InputStorageConfig struct {
- Type string `json:"Type" validate:"required,oneof=b2 s3 local-unix"`
- Config any `json:"Config" validate:"required"`
-}
-
-func (sc *InputStorageConfig) UnmarshalJSON(data []byte) error {
- var tmp struct {
- Type string `json:"Type"`
- Config json.RawMessage `json:"Config"`
- }
-
- if err := json.Unmarshal(data, &tmp); err != nil {
- return err
- }
-
- sc.Type = tmp.Type
-
- switch tmp.Type {
- case "b2":
- var b2Config B2Config
- if err := json.Unmarshal(tmp.Config, &b2Config); err != nil {
- return fmt.Errorf("unmarshal B2Config: %w", err)
- }
- sc.Config = &b2Config
- case "s3":
- var s3Config S3Config
- if err := json.Unmarshal(tmp.Config, &s3Config); err != nil {
- return fmt.Errorf("unmarshal S3Config: %w", err)
- }
- sc.Config = &s3Config
- case "local-unix":
- var localUnixConfig InputLocalUnixConfig
- if err := json.Unmarshal(tmp.Config, &localUnixConfig); err != nil {
- return fmt.Errorf("unmarshal LocalUnixConfig: %w", err)
- }
- sc.Config = &localUnixConfig
- default:
- return fmt.Errorf("unsupported storage type: %s", tmp.Type)
- }
-
- return nil
-}
-
-type ConverterConfig struct {
- Type string `json:"Type" validate:"required,oneof=webp jpeg"`
- Config any `json:"Config" validate:"required"`
+ Input InputConfig `json:"Input" validate:"required"`
Output OutputConfig `json:"Output" validate:"required"`
}
-func (pc *ConverterConfig) UnmarshalJSON(data []byte) error {
- var tmp struct {
- Type string `json:"Type"`
- Config json.RawMessage `json:"Config"`
- Output OutputConfig `json:"Output"`
- }
-
- if err := json.Unmarshal(data, &tmp); err != nil {
- return err
- }
-
- pc.Type = tmp.Type
- pc.Output = tmp.Output
-
- switch tmp.Type {
- case "webp":
- var webpConfig WebpConfig
- if err := json.Unmarshal(tmp.Config, &webpConfig); err != nil {
- return fmt.Errorf("unmarshal WebpConfig: %w", err)
- }
- pc.Config = &webpConfig
- case "jpeg":
- var jpegConfig JpegConfig
- if err := json.Unmarshal(tmp.Config, &jpegConfig); err != nil {
- return fmt.Errorf("unmarshal JpegConfig: %w", err)
- }
- pc.Config = &jpegConfig
- default:
- return fmt.Errorf("unsupported storage type: %s", tmp.Type)
- }
-
- return nil
-}
-
-type WebpConfig struct {
- Quality int `json:"Quality" validate:"required,min=1,max=100"`
- Size SizeConfig `json:"Size"`
-}
-
-type JpegConfig struct {
- ExtensionName string `json:"ExtensionName" validate:"alpha"`
- Quality int `json:"Quality" validate:"required,min=1,max=100"`
- Size SizeConfig `json:"Size"`
+type InputConfig struct {
+ Storage StorageConfig `json:"Storage" validate:"required"`
}
-type SizeConfig struct {
- MaxWidth int `json:"MaxWidth"`
- MaxHeight int `json:"MaxHeight"`
+type OutputConfig struct {
+ Storage StorageConfig `json:"Storage" validate:"required"`
+ Quality int `json:"Quality" validate:"required,min=1,max=100"`
+ Size SizeConfig `json:"Size" validate:"required"`
}
-type OutputStorageConfig struct {
- Type string `json:"Type" validate:"required,oneof=b2 s3 local-unix"`
+type StorageConfig struct {
+ Type string `json:"Type" validate:"required,oneof=b2 local"`
Config any `json:"Config" validate:"required"`
}
-func (sc *OutputStorageConfig) UnmarshalJSON(data []byte) error {
+func (sc *StorageConfig) UnmarshalJSON(data []byte) error {
var tmp struct {
Type string `json:"Type"`
Config json.RawMessage `json:"Config"`
@@ -146,19 +46,13 @@ func (sc *OutputStorageConfig) UnmarshalJSON(data []byte) error {
if err := json.Unmarshal(tmp.Config, &b2Config); err != nil {
return fmt.Errorf("unmarshal B2Config: %w", err)
}
- sc.Config = &b2Config
- case "s3":
- var s3Config S3Config
- if err := json.Unmarshal(tmp.Config, &s3Config); err != nil {
- return fmt.Errorf("unmarshal S3Config: %w", err)
+ sc.Config = b2Config
+ case "local":
+ var localConfig LocalConfig
+ if err := json.Unmarshal(tmp.Config, &localConfig); err != nil {
+ return fmt.Errorf("unmarshal LocalConfig: %w", err)
}
- sc.Config = &s3Config
- case "local-unix":
- var localUnixConfig OutputLocalUnixConfig
- if err := json.Unmarshal(tmp.Config, &localUnixConfig); err != nil {
- return fmt.Errorf("unmarshal LocalUnixConfig: %w", err)
- }
- sc.Config = &localUnixConfig
+ sc.Config = localConfig
default:
return fmt.Errorf("unsupported storage type: %s", tmp.Type)
}
@@ -174,31 +68,13 @@ type B2Config struct {
ApplicationKey string `json:"ApplicationKey"`
}
-type S3Config struct {
- BucketName string `json:"BucketName" validate:"required,min=1"`
- Region string `json:"Region" validate:"required,min=1"`
- Prefix string `json:"Prefix"`
- Endpoint string `json:"Endpoint"`
- UsePathStyle bool `json:"UsePathStyle"`
- AccessKeyID string `json:"AccessKeyID"`
- SecretAccessKey string `json:"SecretAccessKey"`
-}
-
-type InputLocalUnixConfig struct {
- MaxDepth int `json:"MaxDepth" validate:"required,min=0"`
- Path string `json:"Path" validate:"required,min=1"`
-}
-
-type OutputConfig struct {
- RewriteOn string `json:"RewriteOn" validate:"oneof=Never UnequalHashInCache Always"`
- Storage OutputStorageConfig `json:"Storage" validate:"required"`
+type LocalConfig struct {
+ Path string `json:"Path" validate:"required,min=1"`
}
-type OutputLocalUnixConfig struct {
- Path string `json:"Path" validate:"required,min=1,dirpath"`
- DirPermissionMode string `json:"DirPermissionMode" validate:"required,min=3"`
- FilePermissionMode string `json:"FilePermissionMode" validate:"required,min=3"`
- AttributesImplementation string `json:"AttributesImplementation" validate:"required,oneof=xattr none"`
+type SizeConfig struct {
+ MaxWidth int `json:"MaxWidth" validate:"required,min=0"`
+ MaxHeight int `json:"MaxHeight" validate:"required,min=0"`
}
func LoadConfig(path string, config *Config) error {
@@ -213,12 +89,6 @@ func LoadConfig(path string, config *Config) error {
return err
}
- for i := range config.Converters {
- if config.Converters[i].Output.RewriteOn == "" {
- config.Converters[i].Output.RewriteOn = "UnequalHashInCache"
- }
- }
-
return nil
}
diff --git a/config/config.json b/config/config.json
new file mode 100644
index 0000000..d818107
--- /dev/null
+++ b/config/config.json
@@ -0,0 +1,31 @@
+{
+ "Input": {
+ "Storage": {
+ "Type": "b2",
+ "Config": {
+ "BucketName": "sayana-photos",
+ "Region": "eu-central-003",
+ "Prefix": "full/",
+ "KeyID": "${B2_KEY_ID}",
+ "ApplicationKey": "${B2_APPLICATION_KEY}"
+ }
+ }
+ },
+ "Output": {
+ "Storage": {
+ "Type": "b2",
+ "Config": {
+ "BucketName": "sayana-photos",
+ "Region": "eu-central-003",
+ "Prefix": "thumbnails/",
+ "KeyID": "${B2_KEY_ID}",
+ "ApplicationKey": "${B2_APPLICATION_KEY}"
+ }
+ },
+ "Quality": 80,
+ "Size": {
+ "MaxWidth": 1280,
+ "MaxHeight": 0
+ }
+ }
+} \ No newline at end of file
diff --git a/go.mod b/go.mod
index a68df13..5a5a067 100644
--- a/go.mod
+++ b/go.mod
@@ -1,39 +1,16 @@
module github.com/SayaAndy/saya-today-thumbnail-generator
-go 1.26.0
+go 1.24.4
require (
- github.com/Backblaze/blazer v0.7.2
- github.com/go-playground/validator/v10 v10.30.1
- github.com/kolesa-team/go-webp v1.0.5
- golang.org/x/image v0.38.0
- golang.org/x/sys v0.42.0
-)
-
-require (
- github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
- github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
- github.com/aws/aws-sdk-go-v2/config v1.32.14 // indirect
- github.com/aws/aws-sdk-go-v2/credentials v1.19.14 // indirect
- github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect
- github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
- github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
- github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
- github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
- github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 // indirect
- github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect
- github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect
- github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect
- github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect
- github.com/aws/smithy-go v1.24.2 // indirect
- github.com/gabriel-vasile/mimetype v1.4.13 // indirect
+ github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
+ github.com/go-playground/validator/v10 v10.27.0 // indirect
+ github.com/kolesa-team/go-webp v1.0.5 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
- golang.org/x/crypto v0.49.0 // indirect
- golang.org/x/text v0.35.0 // indirect
+ golang.org/x/crypto v0.33.0 // indirect
+ golang.org/x/net v0.34.0 // indirect
+ golang.org/x/sys v0.30.0 // indirect
+ golang.org/x/text v0.22.0 // indirect
)
diff --git a/go.sum b/go.sum
index 1a4b043..5276298 100644
--- a/go.sum
+++ b/go.sum
@@ -1,75 +1,26 @@
-github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM=
-github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10=
-github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
-github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI=
-github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI=
-github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI=
-github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg=
-github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM=
-github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U=
-github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw=
-github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
-github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM=
-github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
-github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
-github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
+github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w=
-github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM=
+github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4=
+github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
github.com/kolesa-team/go-webp v1.0.5 h1:GZQHJBaE8dsNKZltfwqsL0qVJ7vqHXsfA+4AHrQW3pE=
github.com/kolesa-team/go-webp v1.0.5/go.mod h1:QmJu0YHXT3ex+4SgUvs+a+1SFCDcCqyZg+LbIuNNTnE=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
-golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
-golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE=
-golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
-golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go
deleted file mode 100644
index fe94333..0000000
--- a/internal/client/input/b2.go
+++ /dev/null
@@ -1,135 +0,0 @@
-package input
-
-import (
- "context"
- "fmt"
- "io"
- "slices"
- "strings"
-
- "github.com/Backblaze/blazer/b2"
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-var _ InputClient = (*B2InputClient)(nil)
-
-type B2InputClient struct {
- prefix string
- bucket *b2.Bucket
- bucketName string
- b2cl *b2.Client
- knownExtensions []string
-}
-
-func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) {
- if cfg.Storage.Type != "b2" {
- return nil, fmt.Errorf("invalid storage type for B2InputClient")
- }
- b2cfg := cfg.Storage.Config.(*config.B2Config)
-
- b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey)
- if err != nil {
- return nil, err
- }
-
- bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName)
- if err != nil {
- return nil, err
- }
-
- return &B2InputClient{b2cl: b2cl, bucket: bucket, bucketName: b2cfg.BucketName, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
-}
-
-func (c *B2InputClient) Scan() ([]string, error) {
- filePaths := []string{}
-
- iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix))
-
- for iter.Next() {
- obj := iter.Object()
- if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
- }
-
- attrs, err := obj.Attrs(context.Background())
- if err != nil {
- return nil, fmt.Errorf("get attributes for object: %w", err)
- }
-
- if attrs.Status != b2.Uploaded {
- continue
- }
-
- name := obj.Name()
-
- if len(c.knownExtensions) != 0 {
- nameParts := strings.Split(name, ".")
- if len(nameParts) < 2 {
- continue
- }
- ext := strings.ToLower(nameParts[len(nameParts)-1])
- if !slices.Contains(c.knownExtensions, ext) {
- continue
- }
- }
-
- filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
- }
-
- if err := iter.Err(); err != nil {
- return nil, fmt.Errorf("iterate over B2 objects: %w", err)
- }
-
- return filePaths, nil
-}
-
-func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- obj := c.bucket.Object(c.prefix + path)
- if obj == nil {
- return nil, fmt.Errorf("object not found in B2 bucket")
- }
-
- attrs, err := obj.Attrs(context.Background())
- if err != nil {
- return nil, fmt.Errorf("get attributes for object: %w", err)
- }
-
- metadata := MetadataStruct{
- Name: attrs.Name,
- StorageType: "b2",
- Hash: attrs.SHA1,
- ContentType: attrs.ContentType,
- FirstCreated: attrs.UploadTimestamp,
- LastModified: attrs.LastModified,
- Misc: attrs.Info,
- Size: attrs.Size,
- }
-
- switch attrs.Status {
- case b2.Uploaded:
- metadata.Misc["b2-status"] = "Uploaded"
- case b2.Folder:
- metadata.Misc["b2-status"] = "Folder"
- case b2.Hider:
- metadata.Misc["b2-status"] = "Hider"
- case b2.Started:
- metadata.Misc["b2-status"] = "Started"
- default:
- metadata.Misc["b2-status"] = "Unknown"
- }
-
- return &metadata, nil
-}
-
-func (c *B2InputClient) ID(path string) string {
- return fmt.Sprintf("b2://%s/%s%s", c.bucketName, c.prefix, path)
-}
-
-func (c *B2InputClient) GetReader(path string) (io.ReadCloser, error) {
- obj := c.bucket.Object(c.prefix + path)
- if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
- }
-
- return obj.NewReader(context.Background()), nil
-}
diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go
deleted file mode 100644
index dca64db..0000000
--- a/internal/client/input/input_client_interface.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package input
-
-import (
- "io"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-type InputClient interface {
- Scan() ([]string, error)
- ReadMetadata(string) (*MetadataStruct, error)
- GetReader(string) (io.ReadCloser, error)
- ID(path string) string
-}
-
-type MetadataStruct struct {
- Name string
- StorageType string
- Hash string
- ContentType string
- FirstCreated time.Time
- LastModified time.Time
- Size int64
- Misc map[string]string
-}
-
-var NewInputClientMap = map[string]func(cfg *config.InputConfig) (InputClient, error){
- "b2": NewB2InputClient,
- "s3": NewS3InputClient,
- "local-unix": NewLocalUnixInputClient,
-}
diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go
deleted file mode 100644
index 21b08a9..0000000
--- a/internal/client/input/local_unix.go
+++ /dev/null
@@ -1,108 +0,0 @@
-package input
-
-import (
- "fmt"
- "io"
- "log/slog"
- "mime"
- "os"
- "slices"
- "strconv"
- "strings"
- "syscall"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-var _ InputClient = (*LocalUnixInputClient)(nil)
-
-type LocalUnixInputClient struct {
- path string
- maxDepth int
- knownExtensions []string
-}
-
-func NewLocalUnixInputClient(cfg *config.InputConfig) (InputClient, error) {
- if cfg.Storage.Type != "local-unix" {
- return nil, fmt.Errorf("invalid storage type for LocalUnixInputClient")
- }
- localCfg := cfg.Storage.Config.(*config.InputLocalUnixConfig)
-
- return &LocalUnixInputClient{
- path: localCfg.Path,
- maxDepth: localCfg.MaxDepth,
- knownExtensions: cfg.KnownExtensions,
- }, nil
-}
-
-func (c *LocalUnixInputClient) Scan() ([]string, error) {
- return c.recursiveScan(c.path, c.maxDepth)
-}
-
-func (c *LocalUnixInputClient) recursiveScan(dir string, depth int) ([]string, error) {
- filePaths := make([]string, 0)
- entries, err := os.ReadDir(dir)
- if err != nil {
- return nil, fmt.Errorf("fail to read directory: %w", err)
- }
-
- for _, entry := range entries {
- if entry.IsDir() && depth > 0 {
- subFilePaths, err := c.recursiveScan(dir+entry.Name()+"/", depth-1)
- if err != nil {
- return nil, fmt.Errorf("fail to scan subdirectory '%s': %w", entry.Name(), err)
- }
- filePaths = append(filePaths, subFilePaths...)
- } else if !entry.IsDir() {
- nameParts := strings.Split(entry.Name(), ".")
- if len(nameParts) < 2 {
- continue
- }
- if slices.Contains(c.knownExtensions, strings.ToLower(nameParts[len(nameParts)-1])) {
- filePaths = append(filePaths, strings.TrimPrefix(dir+entry.Name(), c.path))
- }
- fmt.Println(entry.Name())
- }
- }
-
- return filePaths, nil
-}
-
-func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- nodePathParts := strings.Split(path, "/")
- nodeName := nodePathParts[len(nodePathParts)-1]
- nodeNameParts := strings.Split(nodeName, ".")
- nodeExt := ""
- if len(nodeNameParts) >= 2 {
- nodeExt = nodeNameParts[len(nodeNameParts)-1]
- }
- slog.Debug("got a file extension", slog.String("extension", nodeExt), slog.String("path", path), slog.String("filename", nodeName))
-
- fileInfo, err := os.Stat(c.path + path)
- if err != nil {
- return nil, fmt.Errorf("fail to read file info: %w", err)
- }
-
- stat_t := fileInfo.Sys().(*syscall.Stat_t)
- creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
-
- return &MetadataStruct{
- Name: fileInfo.Name(),
- StorageType: "local-unix",
- Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16),
- ContentType: mime.TypeByExtension("." + nodeExt),
- FirstCreated: creationTime,
- LastModified: fileInfo.ModTime(),
- Size: fileInfo.Size(),
- Misc: map[string]string{},
- }, nil
-}
-
-func (c *LocalUnixInputClient) ID(path string) string {
- return fmt.Sprintf("local-unix://%s%s", c.path, path)
-}
-
-func (c *LocalUnixInputClient) GetReader(path string) (io.ReadCloser, error) {
- return os.Open(c.path + path)
-}
diff --git a/internal/client/input/s3.go b/internal/client/input/s3.go
deleted file mode 100644
index 00684c2..0000000
--- a/internal/client/input/s3.go
+++ /dev/null
@@ -1,157 +0,0 @@
-package input
-
-import (
- "context"
- "fmt"
- "io"
- "slices"
- "strings"
-
- "github.com/aws/aws-sdk-go-v2/aws"
- awsconfig "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-var _ InputClient = (*S3InputClient)(nil)
-
-type S3InputClient struct {
- prefix string
- bucketName string
- s3cl *s3.Client
- knownExtensions []string
-}
-
-func NewS3InputClient(cfg *config.InputConfig) (InputClient, error) {
- if cfg.Storage.Type != "s3" {
- return nil, fmt.Errorf("invalid storage type for S3InputClient")
- }
- s3cfg := cfg.Storage.Config.(*config.S3Config)
-
- s3cl, err := newS3Client(s3cfg)
- if err != nil {
- return nil, fmt.Errorf("create S3 client: %w", err)
- }
-
- return &S3InputClient{
- s3cl: s3cl,
- bucketName: s3cfg.BucketName,
- prefix: s3cfg.Prefix,
- knownExtensions: cfg.KnownExtensions,
- }, nil
-}
-
-func (c *S3InputClient) Scan() ([]string, error) {
- var filePaths []string
-
- paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
- Bucket: aws.String(c.bucketName),
- Prefix: aws.String(c.prefix),
- })
-
- for paginator.HasMorePages() {
- output, err := paginator.NextPage(context.Background())
- if err != nil {
- return nil, fmt.Errorf("list S3 objects: %w", err)
- }
-
- for _, obj := range output.Contents {
- name := aws.ToString(obj.Key)
-
- if len(c.knownExtensions) != 0 {
- nameParts := strings.Split(name, ".")
- if len(nameParts) < 2 {
- continue
- }
- ext := strings.ToLower(nameParts[len(nameParts)-1])
- if !slices.Contains(c.knownExtensions, ext) {
- continue
- }
- }
-
- filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
- }
- }
-
- return filePaths, nil
-}
-
-func (c *S3InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- key := c.prefix + path
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("head S3 object %s: %w", key, err)
- }
-
- metadata := MetadataStruct{
- Name: key,
- StorageType: "s3",
- Hash: strings.Trim(aws.ToString(head.ETag), "\""),
- ContentType: aws.ToString(head.ContentType),
- Misc: head.Metadata,
- }
-
- if head.ContentLength != nil {
- metadata.Size = *head.ContentLength
- }
- if head.LastModified != nil {
- metadata.LastModified = *head.LastModified
- metadata.FirstCreated = *head.LastModified
- }
-
- return &metadata, nil
-}
-
-func (c *S3InputClient) ID(path string) string {
- return fmt.Sprintf("s3://%s/%s%s", c.bucketName, c.prefix, path)
-}
-
-func (c *S3InputClient) GetReader(path string) (io.ReadCloser, error) {
- key := c.prefix + path
-
- output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("get S3 object %s: %w", key, err)
- }
-
- return output.Body, nil
-}
-
-func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(cfg.Region),
- }
-
- if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
- opts = append(opts, awsconfig.WithCredentialsProvider(
- credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
- ))
- }
-
- awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
- if err != nil {
- return nil, fmt.Errorf("load AWS config: %w", err)
- }
-
- var s3Opts []func(*s3.Options)
- if cfg.Endpoint != "" {
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.BaseEndpoint = aws.String(cfg.Endpoint)
- })
- }
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.UsePathStyle = cfg.UsePathStyle
- o.DisableLogOutputChecksumValidationSkipped = true
- })
-
- return s3.NewFromConfig(awsCfg, s3Opts...), nil
-}
diff --git a/internal/client/output/b2.go b/internal/client/output/b2.go
deleted file mode 100644
index 0616368..0000000
--- a/internal/client/output/b2.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package output
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log/slog"
-
- "github.com/Backblaze/blazer/b2"
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
-)
-
-var _ OutputClient = (*B2OutputClient)(nil)
-
-type B2OutputClient struct {
- prefix string
- bucket *b2.Bucket
- b2cl *b2.Client
-}
-
-func NewB2OutputClient(cfg *config.OutputConfig) (OutputClient, error) {
- if cfg.Storage.Type != "b2" {
- return nil, fmt.Errorf("invalid storage type for B2OutputClient")
- }
- b2cfg := cfg.Storage.Config.(*config.B2Config)
-
- b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey)
- if err != nil {
- return nil, err
- }
-
- bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName)
- if err != nil {
- return nil, err
- }
-
- return &B2OutputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
-}
-
-func (c *B2OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) {
- obj := c.bucket.Object(c.prefix + path)
- if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
- }
-
- attrs := &b2.Attrs{Info: make(map[string]string)}
- attrs.Info["sha1-original"] = inputMetadata.Hash
- attrs.ContentType = outputContentType
-
- return obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs)), nil
-}
-
-func (c *B2OutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- obj := c.bucket.Object(c.prefix + path)
- if obj == nil {
- return nil, fmt.Errorf("failed to reference object in B2 bucket")
- }
-
- attrs, err := obj.Attrs(context.Background())
- if err != nil {
- return nil, fmt.Errorf("get attributes for object: %w", err)
- }
-
- metadata := MetadataStruct{
- Name: attrs.Name,
- StorageType: "b2",
- Hash: attrs.SHA1,
- HashOriginal: attrs.Info["sha1-original"],
- ContentType: attrs.ContentType,
- FirstCreated: attrs.UploadTimestamp,
- LastModified: attrs.LastModified,
- Misc: attrs.Info,
- Size: attrs.Size,
- }
-
- switch attrs.Status {
- case b2.Uploaded:
- metadata.Misc["b2-status"] = "Uploaded"
- case b2.Folder:
- metadata.Misc["b2-status"] = "Folder"
- case b2.Hider:
- metadata.Misc["b2-status"] = "Hider"
- case b2.Started:
- metadata.Misc["b2-status"] = "Started"
- default:
- metadata.Misc["b2-status"] = "Unknown"
- }
-
- return &metadata, nil
-}
-
-func (c *B2OutputClient) IsMissing(path string) bool {
- obj := c.bucket.Object(c.prefix + path)
- if obj == nil {
- return true
- }
-
- attrs, err := obj.Attrs(context.Background())
- if err != nil {
- return true
- }
-
- attrsJson, _ := json.Marshal(attrs)
- slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(attrsJson)))
-
- if attrs.Size == 0 {
- return true
- }
-
- return attrs.Status == b2.Hider
-}
diff --git a/internal/client/output/local_unix.go b/internal/client/output/local_unix.go
deleted file mode 100644
index 6f9a7b3..0000000
--- a/internal/client/output/local_unix.go
+++ /dev/null
@@ -1,129 +0,0 @@
-package output
-
-import (
- "fmt"
- "io"
- "log/slog"
- "mime"
- "os"
- "strconv"
- "strings"
- "syscall"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "golang.org/x/sys/unix"
-)
-
-var _ OutputClient = (*LocalUnixOutputClient)(nil)
-
-type LocalUnixOutputClient struct {
- path string
- fileMode uint32
- dirMode uint32
- attrMode string
-}
-
-func NewLocalUnixOutputClient(cfg *config.OutputConfig) (OutputClient, error) {
- if cfg.Storage.Type != "local-unix" {
- return nil, fmt.Errorf("invalid storage type for LocalUnixOutputClient")
- }
- localCfg := cfg.Storage.Config.(*config.OutputLocalUnixConfig)
-
- fpm, err := strconv.ParseInt(localCfg.FilePermissionMode, 8, 32)
- if err != nil {
- return nil, fmt.Errorf("fail to parse file permission mode as an octal number: %w", err)
- }
-
- dpm, err := strconv.ParseInt(localCfg.DirPermissionMode, 8, 32)
- if err != nil {
- return nil, fmt.Errorf("fail to parse directory permission mode as an octal number: %w", err)
- }
-
- return &LocalUnixOutputClient{localCfg.Path, uint32(fpm), uint32(dpm), localCfg.AttributesImplementation}, nil
-}
-
-func (c *LocalUnixOutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, _ string) (io.WriteCloser, error) {
- pathSegments := strings.Split(path, "/")
- dirpath := strings.Join(pathSegments[0:len(pathSegments)-1], "/")
- if err := os.MkdirAll(c.path+dirpath, os.FileMode(c.dirMode)); err != nil {
- return nil, fmt.Errorf("fail to mkdir parent directories for a path: %w", err)
- }
-
- var err error
- if _, err = os.Stat(c.path + path); err != nil && !os.IsNotExist(err) {
- return nil, fmt.Errorf("fail to stat a file, although file exists: %w", err)
- }
- if os.IsNotExist(err) {
- f, err := os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE, os.FileMode(c.fileMode))
- if err != nil {
- return nil, fmt.Errorf("fail to create a file: %w", err)
- }
- f.Close()
- }
-
- switch c.attrMode {
- case "xattr":
- if err := unix.Setxattr(c.path+path, "user.originalfile.mddate", []byte(strconv.FormatInt(inputMetadata.LastModified.Unix(), 16)), 0); err != nil {
- return nil, fmt.Errorf("fail to write user.originalfile.mddate xattribute: %w", err)
- }
- case "none":
- default:
- return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode)
- }
-
- return os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(c.fileMode))
-}
-
-func (c *LocalUnixOutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- nodePathParts := strings.Split(path, "/")
- nodeName := nodePathParts[len(nodePathParts)-1]
- nodeNameParts := strings.Split(nodeName, ".")
- nodeExt := ""
- if len(nodeNameParts) >= 2 {
- nodeExt = nodeNameParts[len(nodeNameParts)-1]
- }
-
- fileInfo, err := os.Stat(c.path + path)
- if err != nil {
- return nil, fmt.Errorf("fail to read file info: %w", err)
- }
-
- stat_t := fileInfo.Sys().(*syscall.Stat_t)
- creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
-
- mddateOriginal := make([]byte, 0)
- switch c.attrMode {
- case "xattr":
- sz, err := unix.Getxattr(c.path+path, "user.originalfile.mddate", nil)
- if err != nil {
- slog.Warn("fail to get size of user.originalfile.mddate attribute, proceeding as if no such attribute is there", slog.String("error", err.Error()))
- break
- }
- mddateOriginal = make([]byte, sz)
- if _, err = unix.Getxattr(c.path+path, "user.originalfile.mddate", mddateOriginal); err != nil {
- return nil, fmt.Errorf("fail to get user.originalfile.mddate attribute: %w", err)
- }
- case "none":
- default:
- return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode)
- }
-
- return &MetadataStruct{
- Name: fileInfo.Name(),
- StorageType: "local-unix",
- Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16),
- HashOriginal: string(mddateOriginal),
- ContentType: mime.TypeByExtension("." + nodeExt),
- FirstCreated: creationTime,
- LastModified: fileInfo.ModTime(),
- Size: fileInfo.Size(),
- Misc: map[string]string{},
- }, nil
-}
-
-func (c *LocalUnixOutputClient) IsMissing(path string) bool {
- _, err := os.Stat(c.path + path)
- return err != nil
-}
diff --git a/internal/client/output/output_client_interface.go b/internal/client/output/output_client_interface.go
deleted file mode 100644
index 4415aa4..0000000
--- a/internal/client/output/output_client_interface.go
+++ /dev/null
@@ -1,33 +0,0 @@
-package output
-
-import (
- "io"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
-)
-
-type OutputClient interface {
- GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error)
- ReadMetadata(path string) (*MetadataStruct, error)
- IsMissing(path string) bool
-}
-
-type MetadataStruct struct {
- Name string
- StorageType string
- Hash string
- HashOriginal string
- ContentType string
- FirstCreated time.Time
- LastModified time.Time
- Size int64
- Misc map[string]string
-}
-
-var NewOutputClientMap = map[string]func(cfg *config.OutputConfig) (OutputClient, error){
- "b2": NewB2OutputClient,
- "s3": NewS3OutputClient,
- "local-unix": NewLocalUnixOutputClient,
-}
diff --git a/internal/client/output/s3.go b/internal/client/output/s3.go
deleted file mode 100644
index f0527fb..0000000
--- a/internal/client/output/s3.go
+++ /dev/null
@@ -1,170 +0,0 @@
-package output
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log/slog"
- "strings"
-
- "github.com/aws/aws-sdk-go-v2/aws"
- awsconfig "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
-)
-
-var _ OutputClient = (*S3OutputClient)(nil)
-
-type S3OutputClient struct {
- prefix string
- bucketName string
- s3cl *s3.Client
-}
-
-func NewS3OutputClient(cfg *config.OutputConfig) (OutputClient, error) {
- if cfg.Storage.Type != "s3" {
- return nil, fmt.Errorf("invalid storage type for S3OutputClient")
- }
- s3cfg := cfg.Storage.Config.(*config.S3Config)
-
- s3cl, err := newS3Client(s3cfg)
- if err != nil {
- return nil, fmt.Errorf("create S3 client: %w", err)
- }
-
- return &S3OutputClient{
- s3cl: s3cl,
- bucketName: s3cfg.BucketName,
- prefix: s3cfg.Prefix,
- }, nil
-}
-
-func (c *S3OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) {
- key := c.prefix + path
-
- return &s3WriteCloser{
- key: key,
- bucketName: c.bucketName,
- s3cl: c.s3cl,
- contentType: outputContentType,
- hashOriginal: inputMetadata.Hash,
- buf: &bytes.Buffer{},
- }, nil
-}
-
-func (c *S3OutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- key := c.prefix + path
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("head S3 object %s: %w", key, err)
- }
-
- metadata := MetadataStruct{
- Name: key,
- StorageType: "s3",
- Hash: strings.Trim(aws.ToString(head.ETag), "\""),
- HashOriginal: head.Metadata["sha1-original"],
- ContentType: aws.ToString(head.ContentType),
- Misc: head.Metadata,
- }
-
- if head.ContentLength != nil {
- metadata.Size = *head.ContentLength
- }
- if head.LastModified != nil {
- metadata.LastModified = *head.LastModified
- metadata.FirstCreated = *head.LastModified
- }
-
- return &metadata, nil
-}
-
-func (c *S3OutputClient) IsMissing(path string) bool {
- key := c.prefix + path
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return true
- }
-
- headJson, _ := json.Marshal(head)
- slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(headJson)))
-
- if head.ContentLength == nil || *head.ContentLength == 0 {
- return true
- }
-
- return false
-}
-
-// s3WriteCloser buffers writes and uploads to S3 on Close.
-type s3WriteCloser struct {
- key string
- bucketName string
- s3cl *s3.Client
- contentType string
- hashOriginal string
- buf *bytes.Buffer
-}
-
-func (w *s3WriteCloser) Write(p []byte) (int, error) {
- return w.buf.Write(p)
-}
-
-func (w *s3WriteCloser) Close() error {
- _, err := w.s3cl.PutObject(context.Background(), &s3.PutObjectInput{
- Bucket: aws.String(w.bucketName),
- Key: aws.String(w.key),
- Body: bytes.NewReader(w.buf.Bytes()),
- ContentType: aws.String(w.contentType),
- Metadata: map[string]string{
- "sha1-original": w.hashOriginal,
- },
- })
- if err != nil {
- return fmt.Errorf("put S3 object %s: %w", w.key, err)
- }
- return nil
-}
-
-func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(cfg.Region),
- }
-
- if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
- opts = append(opts, awsconfig.WithCredentialsProvider(
- credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
- ))
- }
-
- awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
- if err != nil {
- return nil, fmt.Errorf("load AWS config: %w", err)
- }
-
- var s3Opts []func(*s3.Options)
- if cfg.Endpoint != "" {
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.BaseEndpoint = aws.String(cfg.Endpoint)
- })
- }
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.UsePathStyle = cfg.UsePathStyle
- o.DisableLogOutputChecksumValidationSkipped = true
- })
-
- return s3.NewFromConfig(awsCfg, s3Opts...), nil
-}
diff --git a/internal/converter/converter_interface.go b/internal/converter/converter_interface.go
deleted file mode 100644
index abff99d..0000000
--- a/internal/converter/converter_interface.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package converter
-
-import (
- "io"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output"
-)
-
-type Converter interface {
- Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error
- DeductOutputPath(inputPath string) string
- ReadMetadata(path string) (*output.MetadataStruct, error)
- IsMissing(path string) bool
-}
-
-var NewConverterMap = map[string]func(cfg *config.ConverterConfig) (Converter, error){
- "webp": NewWebpConverter,
- "jpeg": NewJpegConverter,
-}
diff --git a/internal/converter/jpeg.go b/internal/converter/jpeg.go
deleted file mode 100644
index a35a47c..0000000
--- a/internal/converter/jpeg.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package converter
-
-import (
- "fmt"
- "image"
- "image/jpeg"
- "image/png"
- "io"
- "log/slog"
- "path/filepath"
- "strings"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output"
- "github.com/kolesa-team/go-webp/webp"
- "golang.org/x/image/draw"
-)
-
-var _ Converter = (*JpegConverter)(nil)
-
-type JpegConverter struct {
- maxWidth int
- maxHeight int
- extensionName string
- quality int
- outputClient output.OutputClient
-}
-
-func NewJpegConverter(cfg *config.ConverterConfig) (Converter, error) {
- if cfg.Type != "jpeg" {
- return nil, fmt.Errorf("invalid storage type for JpegConverter")
- }
- jpegCfg := cfg.Config.(*config.JpegConfig)
-
- outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output)
- if err != nil {
- return nil, fmt.Errorf("fail to initialize output client: %w", err)
- }
-
- extensionName := ".jpg"
- if jpegCfg.ExtensionName != "" {
- extensionName = "." + jpegCfg.ExtensionName
- }
-
- return &JpegConverter{jpegCfg.Size.MaxWidth, jpegCfg.Size.MaxHeight, extensionName, jpegCfg.Quality, outputClient}, nil
-}
-
-func (p *JpegConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error {
- var src image.Image
-
- writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/jpeg")
- if err != nil {
- return fmt.Errorf("fail to initialize writer for output: %w", err)
- }
- defer writer.Close()
-
- switch inputMetadata.ContentType {
- case "image/jpeg":
- src, err = jpeg.Decode(reader)
- if err != nil {
- return fmt.Errorf("decode jpeg: %w", err)
- }
- case "image/png":
- src, err = png.Decode(reader)
- if err != nil {
- return fmt.Errorf("decode png: %w", err)
- }
- case "image/webp":
- src, err = webp.Decode(reader, nil)
- if err != nil {
- return fmt.Errorf("decode webp: %w", err)
- }
- default:
- return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType)
- }
-
- xCoef := 1.0
- if p.maxWidth > 0 {
- xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X)
- }
- yCoef := 1.0
- if p.maxHeight > 0 {
- yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y)
- }
- slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef))
-
- minCoef := xCoef
- if yCoef < minCoef {
- minCoef = yCoef
- }
-
- if minCoef >= 1.0 {
- return jpeg.Encode(writer, src, &jpeg.Options{Quality: p.quality})
- }
-
- dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5)))
- draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil)
-
- return jpeg.Encode(writer, dst, &jpeg.Options{Quality: p.quality})
-}
-
-func (p *JpegConverter) DeductOutputPath(inputPath string) string {
- withoutExt, _ := strings.CutSuffix(inputPath, filepath.Ext(inputPath))
- return withoutExt + p.extensionName
-}
-
-func (p *JpegConverter) ReadMetadata(path string) (*output.MetadataStruct, error) {
- return p.outputClient.ReadMetadata(path)
-}
-
-func (p *JpegConverter) IsMissing(path string) bool {
- return p.outputClient.IsMissing(path)
-}
diff --git a/internal/converter/webp.go b/internal/converter/webp.go
deleted file mode 100644
index fbffd01..0000000
--- a/internal/converter/webp.go
+++ /dev/null
@@ -1,118 +0,0 @@
-package converter
-
-import (
- "fmt"
- "image"
- "image/jpeg"
- "image/png"
- "io"
- "log/slog"
- "strings"
-
- "golang.org/x/image/draw"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output"
- "github.com/kolesa-team/go-webp/encoder"
- "github.com/kolesa-team/go-webp/webp"
-)
-
-var _ Converter = (*WebpConverter)(nil)
-
-type WebpConverter struct {
- maxWidth int
- maxHeight int
- quality int
- outputClient output.OutputClient
-}
-
-func NewWebpConverter(cfg *config.ConverterConfig) (Converter, error) {
- if cfg.Type != "webp" {
- return nil, fmt.Errorf("invalid storage type for WebpConverter")
- }
- webpCfg := cfg.Config.(*config.WebpConfig)
-
- outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output)
- if err != nil {
- return nil, fmt.Errorf("fail to initialize output client: %w", err)
- }
-
- return &WebpConverter{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality, outputClient}, nil
-}
-
-func (p *WebpConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error {
- var src image.Image
-
- writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/webp")
- if err != nil {
- return fmt.Errorf("fail to initialize writer for output: %w", err)
- }
- defer writer.Close()
-
- switch inputMetadata.ContentType {
- case "image/jpeg":
- src, err = jpeg.Decode(reader)
- if err != nil {
- return fmt.Errorf("decode jpeg: %w", err)
- }
- case "image/png":
- src, err = png.Decode(reader)
- if err != nil {
- return fmt.Errorf("decode png: %w", err)
- }
- case "image/webp":
- src, err = webp.Decode(reader, nil)
- if err != nil {
- return fmt.Errorf("decode webp: %w", err)
- }
- default:
- return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType)
- }
-
- opts, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, float32(p.quality))
- if err != nil {
- return fmt.Errorf("create webp encoder options: %w", err)
- }
-
- xCoef := 1.0
- if p.maxWidth > 0 {
- xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X)
- }
- yCoef := 1.0
- if p.maxHeight > 0 {
- yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y)
- }
- slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef))
-
- minCoef := xCoef
- if yCoef < minCoef {
- minCoef = yCoef
- }
-
- if minCoef >= 1.0 {
- return webp.Encode(writer, src, opts)
- }
-
- dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5)))
- draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil)
-
- return webp.Encode(writer, dst, opts)
-}
-
-func (p *WebpConverter) DeductOutputPath(inputPath string) string {
- pathParts := strings.Split(inputPath, ".")
- if len(pathParts) < 2 {
- return inputPath + ".webp"
- }
- pathParts[len(pathParts)-1] = "webp"
- return strings.Join(pathParts, ".")
-}
-
-func (p *WebpConverter) ReadMetadata(path string) (*output.MetadataStruct, error) {
- return p.outputClient.ReadMetadata(path)
-}
-
-func (p *WebpConverter) IsMissing(path string) bool {
- return p.outputClient.IsMissing(path)
-}
diff --git a/main.go b/main.go
index f3c1196..f03ff52 100644
--- a/main.go
+++ b/main.go
@@ -1,302 +1,20 @@
package main
import (
- "bytes"
- "encoding/csv"
- "encoding/json"
"flag"
- "hash/crc32"
- "io"
- "log/slog"
- "os"
- "os/signal"
- "strconv"
- "strings"
- "sync"
- "syscall"
"github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/converter"
)
var (
- configPath = flag.String("c", "config.json", "Path to the configuration file")
- sigTermChan = make(chan os.Signal, 1)
- cacheMapMutex = &sync.RWMutex{}
- cacheMap = make(map[string]map[uint32]struct{})
+ configPath = flag.String("c", "config.json", "Path to the configuration file")
)
func main() {
- signal.Notify(sigTermChan, os.Interrupt, syscall.SIGTERM)
-
flag.Parse()
cfg := &config.Config{}
if err := config.LoadConfig(*configPath, cfg); err != nil {
- slog.Error("fail to load configuration", slog.String("error", err.Error()))
- os.Exit(1)
- }
-
- slog.SetLogLoggerLevel(cfg.LogLevel)
- slog.Info("starting thumbnail generator...")
-
- select {
- case <-sigTermChan:
- slog.Info("exiting due to termination signal")
- os.Exit(130)
- default:
- }
-
- inputClient, err := input.NewInputClientMap[cfg.Input.Storage.Type](&cfg.Input)
- if err != nil {
- slog.Error("fail to initialize input client", slog.String("error", err.Error()))
- os.Exit(1)
- }
-
- converters := make([]converter.Converter, 0, len(cfg.Converters))
- converterTypes := make([]string, 0, len(cfg.Converters))
- converterHashes := make([]uint32, 0, len(cfg.Converters))
- for _, converterCfg := range cfg.Converters {
- converterBytes, _ := json.Marshal(converterCfg)
- conv, err := converter.NewConverterMap[converterCfg.Type](&converterCfg)
- if err != nil {
- slog.Error("fail to initialize converter", slog.String("error", err.Error()))
- os.Exit(1)
- }
- converters = append(converters, conv)
- converterTypes = append(converterTypes, converterCfg.Type)
- converterHashes = append(converterHashes, crc32.ChecksumIEEE(converterBytes))
- }
-
- generalLogger := slog.With(slog.String("input_storage", cfg.Input.Storage.Type))
- generalLogger.Info("initialized input client and converters", slog.String("converter_types", strings.Join(converterTypes, " ")))
-
- select {
- case <-sigTermChan:
- generalLogger.Info("exiting due to termination signal")
- os.Exit(130)
- default:
- }
-
- files, err := inputClient.Scan()
- if err != nil {
- generalLogger.Error("fail to scan input files", slog.String("error", err.Error()))
- os.Exit(1)
- }
- fileCount := len(files)
- generalLogger.Info("scanned files", slog.Int("file_count", fileCount))
-
- select {
- case <-sigTermChan:
- generalLogger.Info("exiting due to termination signal")
- os.Exit(130)
- default:
- }
-
- if cfg.Input.CacheProcessed {
- cacheFile, err := os.OpenFile(cfg.Input.CacheProcessedCsvPath, os.O_CREATE|os.O_RDONLY, 0644)
- if err != nil {
- generalLogger.Error("fail to initialize cache file", slog.String("cache_path", cfg.Input.CacheProcessedCsvPath), slog.String("error", err.Error()))
- os.Exit(1)
- }
- defer cacheFile.Close()
-
- csvReader := csv.NewReader(cacheFile)
- for {
- rec, err := csvReader.Read()
- if err == io.EOF {
- break
- }
- if err != nil {
- generalLogger.Error("fail to initialize cache while reading file", slog.String("cache_path", cfg.Input.CacheProcessedCsvPath), slog.String("error", err.Error()))
- os.Exit(1)
- }
- if len(rec) != 2 {
- generalLogger.Error("fail to initialize cache while reading file", slog.Int("record_length", len(rec)), slog.String("error", "incorrect format: expected '{image-name},{semicolon-separated-processor-hashes}'"))
- os.Exit(1)
- }
- hashes := strings.Split(rec[1], ";")
- cacheMap[rec[0]] = make(map[uint32]struct{})
- for _, hash := range hashes {
- hashUint, err := strconv.ParseUint(hash, 10, 32)
- if err != nil {
- generalLogger.Error("fail to initialize cache while reading file", slog.Int("record_length", len(rec)), slog.String("error", err.Error()))
- }
- cacheMap[rec[0]][uint32(hashUint)] = struct{}{}
- }
- }
- }
-
- select {
- case <-sigTermChan:
- generalLogger.Info("exiting due to termination signal")
- os.Exit(130)
- default:
- }
-
- processSemaphore := make(chan struct{}, cfg.MaxProcessThreads)
- queueSemaphore := make(chan struct{}, cfg.MaxPreProcessThreads)
- var wg sync.WaitGroup
- wg.Add(fileCount)
-
- processTerminating := false
-
- for i, file := range files {
- queueSemaphore <- struct{}{}
-
- select {
- case <-sigTermChan:
- generalLogger.Info("exiting due to termination signal")
- processTerminating = true
- default:
- }
-
- go func(index int, inputName string, earlyTerminate bool) {
- defer func() { <-queueSemaphore; wg.Done() }()
-
- fileLogger := generalLogger.With(slog.String("input_path", inputName), slog.Int("file_index", index))
-
- if earlyTerminate {
- fileLogger.Info("skip processing file (process is terminating)")
- return
- }
-
- var inputMetadata *input.MetadataStruct
-
- id := inputClient.ID(file)
- cacheMapMutex.Lock()
- if _, ok := cacheMap[id]; !ok {
- cacheMap[id] = make(map[uint32]struct{})
- }
- cacheMapMutex.Unlock()
-
- convertersToLaunch := []int{}
- for j, conv := range converters {
- if cfg.Input.CacheProcessed {
- cacheMapMutex.RLock()
- if _, ok := cacheMap[id][converterHashes[j]]; ok {
- cacheMapMutex.RUnlock()
- fileLogger.Info("skip already processed file (based on cache file containing it and processor)",
- slog.String("file_id", id),
- slog.Uint64("conv_hash", uint64(converterHashes[j])),
- slog.Int("conv_index", j))
- continue
- }
- cacheMapMutex.RUnlock()
- }
- outputName := conv.DeductOutputPath(inputName)
- originalInputHash := ""
- convLogger := fileLogger.With(slog.String("output_path", outputName), slog.Int("conv_index", j))
-
- if inputMetadata == nil {
- inputMetadata, err = inputClient.ReadMetadata(inputName)
- if err != nil {
- fileLogger.Warn("fail to read metadata of (supposedly existing) input file", slog.String("error", err.Error()))
- return
- }
- }
-
- switch cfg.Converters[j].Output.RewriteOn {
- case "Never":
- if !conv.IsMissing(outputName) {
- convLogger.Info("skip already existing file")
- continue
- }
- case "UnequalHashInCache":
- if !conv.IsMissing(outputName) {
- outputMetadata, err := conv.ReadMetadata(outputName)
- if err != nil {
- convLogger.Warn("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error()))
- continue
- }
- originalInputHash = outputMetadata.HashOriginal
- if inputMetadata.Hash == originalInputHash {
- convLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash))
- cacheMapMutex.Lock()
- cacheMap[id][converterHashes[j]] = struct{}{}
- cacheMapMutex.Unlock()
- continue
- }
- }
- case "Always":
- }
-
- convertersToLaunch = append(convertersToLaunch, j)
- }
-
- if len(convertersToLaunch) == 0 {
- return
- }
-
- processSemaphore <- struct{}{}
- defer func() { <-processSemaphore }()
-
- fileLogger.Info("start to process file", slog.String("input_hash", inputMetadata.Hash))
- reader, err := inputClient.GetReader(inputName)
- if err != nil {
- fileLogger.Warn("fail to get reader for input file", slog.String("error", err.Error()))
- return
- }
-
- fileContent, err := io.ReadAll(reader)
- if err != nil {
- fileLogger.Warn("fail to read content of input file", slog.String("error", err.Error()))
- return
- } else if int64(len(fileContent)) != inputMetadata.Size {
- fileLogger.Warn("the downloaded file seems to be missing some of its content",
- slog.Int("actual_size_bytes", len(fileContent)),
- slog.Int64("expected_size_bytes", inputMetadata.Size))
- }
- reader.Close()
-
- for _, convIndex := range convertersToLaunch {
- conv := converters[convIndex]
- outputName := conv.DeductOutputPath(inputName)
- convLogger := fileLogger.With(slog.String("output_path", outputName), slog.Int("conv_index", convIndex))
-
- if err := conv.Process(inputMetadata, bytes.NewReader(fileContent), outputName); err != nil {
- convLogger.Warn("fail to convert file", slog.String("error", err.Error()))
- return
- }
-
- convLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash))
- cacheMapMutex.Lock()
- cacheMap[id][converterHashes[convIndex]] = struct{}{}
- cacheMapMutex.Unlock()
- }
- }(i, file, processTerminating)
- }
-
- wg.Wait()
-
- select {
- case <-sigTermChan:
- generalLogger.Info("exiting due to termination signal")
- os.Exit(130)
- default:
- }
-
- generalLogger.Info("all files processed successfully")
-
- if cfg.Input.CacheProcessed {
- generalLogger.Info("writing cache file")
- cacheFile, err := os.OpenFile(cfg.Input.CacheProcessedCsvPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
- if err != nil {
- generalLogger.Error("error writing cache into file")
- } else {
- for id, hashes := range cacheMap {
- var hashStrings = make([]string, 0, len(hashes))
- for hash := range hashes {
- hashStrings = append(hashStrings, strconv.FormatUint(uint64(hash), 10))
- }
- cacheFile.Write([]byte(id))
- cacheFile.Write([]byte{','})
- cacheFile.WriteString(strings.Join(hashStrings, ";"))
- cacheFile.Write([]byte{'\n'})
- }
- }
+ panic(err)
}
-
- os.Exit(0)
}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-10 13:46:02 +0000