From 373ec07b40178cb3b704aa83fc4fd53abfe279bc Mon Sep 17 00:00:00 2001 From: SayaAndy Date: Tue, 15 Jul 2025 12:43:28 +0700 Subject: feat: prepare all internal pkg (input/output client & processor) --- config/config.go | 58 ++++++++-- config/config.json | 22 +++- go.mod | 5 +- go.sum | 8 ++ internal/client/input/b2.go | 129 ++++++++++++++++++++++ internal/client/input/input_client_interface.go | 11 ++ internal/client/output/b2.go | 95 ++++++++++++++++ internal/client/output/output_client_interface.go | 11 ++ internal/processor/processor_interface.go | 8 ++ internal/processor/webp.go | 81 ++++++++++++++ 10 files changed, 410 insertions(+), 18 deletions(-) create mode 100644 internal/client/input/b2.go create mode 100644 internal/client/input/input_client_interface.go create mode 100644 internal/client/output/b2.go create mode 100644 internal/client/output/output_client_interface.go create mode 100644 internal/processor/processor_interface.go create mode 100644 internal/processor/webp.go diff --git a/config/config.go b/config/config.go index 6d37904..5df29df 100644 --- a/config/config.go +++ b/config/config.go @@ -9,18 +9,14 @@ import ( ) type Config struct { - Input InputConfig `json:"Input" validate:"required"` - Output OutputConfig `json:"Output" validate:"required"` + Input InputConfig `json:"Input" validate:"required"` + Processor ProcessorConfig `json:"Processor" validate:"required"` + Output OutputConfig `json:"Output" validate:"required"` } type InputConfig struct { - Storage StorageConfig `json:"Storage" validate:"required"` -} - -type OutputConfig struct { - Storage StorageConfig `json:"Storage" validate:"required"` - Quality int `json:"Quality" validate:"required,min=1,max=100"` - Size SizeConfig `json:"Size" validate:"required"` + Storage StorageConfig `json:"Storage" validate:"required"` + KnownExtensions []string `json:"KnownExtensions" validate:"required,min=0,dive,min=1"` } type StorageConfig struct { @@ -46,13 +42,13 @@ func (sc *StorageConfig) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(tmp.Config, &b2Config); err != nil { return fmt.Errorf("unmarshal B2Config: %w", err) } - sc.Config = b2Config + sc.Config = &b2Config case "local": var localConfig LocalConfig if err := json.Unmarshal(tmp.Config, &localConfig); err != nil { return fmt.Errorf("unmarshal LocalConfig: %w", err) } - sc.Config = localConfig + sc.Config = &localConfig default: return fmt.Errorf("unsupported storage type: %s", tmp.Type) } @@ -72,11 +68,51 @@ type LocalConfig struct { Path string `json:"Path" validate:"required,min=1"` } +type ProcessorConfig struct { + Type string `json:"Type" validate:"required,oneof=webp"` + Config any `json:"Config" validate:"required"` +} + +func (pc *ProcessorConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + pc.Type = tmp.Type + + switch tmp.Type { + case "webp": + var webpConfig WebpConfig + if err := json.Unmarshal(tmp.Config, &webpConfig); err != nil { + return fmt.Errorf("unmarshal WebpConfig: %w", err) + } + pc.Config = &webpConfig + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type WebpConfig struct { + Quality int `json:"Quality" validate:"required,min=1,max=100"` + Size SizeConfig `json:"Size" validate:"required"` +} + type SizeConfig struct { MaxWidth int `json:"MaxWidth" validate:"required,min=0"` MaxHeight int `json:"MaxHeight" validate:"required,min=0"` } +type OutputConfig struct { + Storage StorageConfig `json:"Storage" validate:"required"` +} + func LoadConfig(path string, config *Config) error { fileBytes, err := os.ReadFile(path) if err != nil { diff --git a/config/config.json b/config/config.json index d818107..298a03f 100644 --- a/config/config.json +++ b/config/config.json @@ -9,6 +9,21 @@ "KeyID": "${B2_KEY_ID}", "ApplicationKey": "${B2_APPLICATION_KEY}" } + }, + "KnownExtensions": [ + "jpg", + "jpeg", + "png" + ] + }, + "Processor": { + "Type": "webp", + "Config": { + "Quality": 80, + "Size": { + "MaxWidth": 1280, + "MaxHeight": 0 + } } }, "Output": { @@ -21,11 +36,6 @@ "KeyID": "${B2_KEY_ID}", "ApplicationKey": "${B2_APPLICATION_KEY}" } - }, - "Quality": 80, - "Size": { - "MaxWidth": 1280, - "MaxHeight": 0 } } -} \ No newline at end of file +} diff --git a/go.mod b/go.mod index 5a5a067..fcfa935 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/SayaAndy/saya-today-thumbnail-generator go 1.24.4 require ( + github.com/Backblaze/blazer v0.7.2 // indirect + github.com/benbusby/b2 v1.4.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -10,7 +12,8 @@ require ( github.com/kolesa-team/go-webp v1.0.5 // indirect github.com/leodido/go-urn v1.4.0 // indirect golang.org/x/crypto v0.33.0 // indirect + golang.org/x/image v0.29.0 // indirect golang.org/x/net v0.34.0 // indirect golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/text v0.27.0 // indirect ) diff --git a/go.sum b/go.sum index 5276298..2f0c3d2 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM= +github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10= +github.com/benbusby/b2 v1.4.0 h1:TbbSskomOrJhJUzOSo3EnU/FCveGwBOk5GrefxdINS0= +github.com/benbusby/b2 v1.4.0/go.mod h1:33DCcJUrJLjGlFT1wLIxzg+/oVv4TlYBnbdlfazKnTg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= @@ -16,11 +20,15 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/image v0.29.0 h1:HcdsyR4Gsuys/Axh0rDEmlBmB68rW1U9BUdB3UVHsas= +golang.org/x/image v0.29.0/go.mod h1:RVJROnf3SLK8d26OW91j4FrIHGbsJ8QnbEocVTOWQDA= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go new file mode 100644 index 0000000..67fb14e --- /dev/null +++ b/internal/client/input/b2.go @@ -0,0 +1,129 @@ +package input + +import ( + "context" + "fmt" + "io" + "slices" + "strings" + + "github.com/Backblaze/blazer/b2" + "github.com/SayaAndy/saya-today-thumbnail-generator/config" +) + +var _ InputClient = (*B2InputClient)(nil) + +type B2InputClient struct { + prefix string + bucket *b2.Bucket + b2cl *b2.Client + knownExtensions []string +} + +func NewB2InputClient(cfg *config.InputConfig) (*B2InputClient, error) { + if cfg.Storage.Type != "b2" { + return nil, fmt.Errorf("invalid storage type for B2InputClient") + } + b2cfg := cfg.Storage.Config.(*config.B2Config) + + b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) + if err != nil { + return nil, err + } + + bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) + if err != nil { + return nil, err + } + + return &B2InputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil +} + +func (c *B2InputClient) Scan() ([]string, error) { + filePaths := []string{} + + iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix)) + + for iter.Next() { + obj := iter.Object() + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + if attrs.Status != b2.Uploaded { + continue + } + + name := obj.Name() + + if len(c.knownExtensions) != 0 { + nameParts := strings.Split(name, ".") + if len(nameParts) < 2 { + continue + } + ext := strings.ToLower(nameParts[len(nameParts)-1]) + if !slices.Contains(c.knownExtensions, ext) { + continue + } + } + + filePaths = append(filePaths, name) + } + + if err := iter.Err(); err != nil { + return nil, fmt.Errorf("iterate over B2 objects: %w", err) + } + + return filePaths, nil +} + +func (c *B2InputClient) ReadMetadata(path string) (map[string]string, error) { + obj := c.bucket.Object(path) + if obj == nil { + return nil, fmt.Errorf("object not found in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + metadata := attrs.Info + metadata["Name"] = attrs.Name + metadata["Size"] = fmt.Sprintf("%d", attrs.Size) + metadata["ContentType"] = attrs.ContentType + metadata["LastModified"] = attrs.LastModified.Format("2006-01-02T15:04:05Z") + metadata["SHA1"] = attrs.SHA1 + metadata["UploadTimestamp"] = attrs.UploadTimestamp.Format("2006-01-02T15:04:05Z") + + switch attrs.Status { + case b2.Uploaded: + metadata["Status"] = "Uploaded" + case b2.Folder: + metadata["Status"] = "Folder" + case b2.Hider: + metadata["Status"] = "Hider" + case b2.Started: + metadata["Status"] = "Started" + default: + metadata["Status"] = "Unknown" + } + + return metadata, nil +} + +func (c *B2InputClient) GetReader(path string) (io.Reader, error) { + obj := c.bucket.Object(path) + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + reader := obj.NewReader(context.Background()) + + return reader, nil +} diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go new file mode 100644 index 0000000..4bbbae5 --- /dev/null +++ b/internal/client/input/input_client_interface.go @@ -0,0 +1,11 @@ +package input + +import ( + "io" +) + +type InputClient interface { + Scan() ([]string, error) + ReadMetadata(string) (map[string]string, error) + GetReader(string) (io.Reader, error) +} diff --git a/internal/client/output/b2.go b/internal/client/output/b2.go new file mode 100644 index 0000000..3c8b53c --- /dev/null +++ b/internal/client/output/b2.go @@ -0,0 +1,95 @@ +package output + +import ( + "context" + "fmt" + "io" + + "github.com/Backblaze/blazer/b2" + "github.com/SayaAndy/saya-today-thumbnail-generator/config" +) + +var _ OutputClient = (*B2OutputClient)(nil) + +type B2OutputClient struct { + prefix string + bucket *b2.Bucket + b2cl *b2.Client +} + +func NewB2OutputClient(cfg *config.OutputConfig) (*B2OutputClient, error) { + if cfg.Storage.Type != "b2" { + return nil, fmt.Errorf("invalid storage type for B2OutputClient") + } + b2cfg := cfg.Storage.Config.(*config.B2Config) + + b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) + if err != nil { + return nil, err + } + + bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) + if err != nil { + return nil, err + } + + return &B2OutputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil +} + +func (c *B2OutputClient) GetWriter(path string) (io.Writer, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + return obj.NewWriter(context.Background()), nil +} + +func (c *B2OutputClient) ReadMetadata(path string) (map[string]string, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + metadata := attrs.Info + metadata["Name"] = attrs.Name + metadata["Size"] = fmt.Sprintf("%d", attrs.Size) + metadata["ContentType"] = attrs.ContentType + metadata["LastModified"] = attrs.LastModified.Format("2006-01-02T15:04:05Z") + metadata["SHA1"] = attrs.SHA1 + metadata["UploadTimestamp"] = attrs.UploadTimestamp.Format("2006-01-02T15:04:05Z") + + switch attrs.Status { + case b2.Uploaded: + metadata["Status"] = "Uploaded" + case b2.Folder: + metadata["Status"] = "Folder" + case b2.Hider: + metadata["Status"] = "Hider" + case b2.Started: + metadata["Status"] = "Started" + default: + metadata["Status"] = "Unknown" + } + + return metadata, nil +} + +func (c *B2OutputClient) IsMissing(path string) (bool, error) { + obj := c.bucket.Object(c.prefix + path) + if obj == nil { + return false, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return true, fmt.Errorf("get attributes for object: %w", err) + } + + return attrs.Status == b2.Hider, nil +} diff --git a/internal/client/output/output_client_interface.go b/internal/client/output/output_client_interface.go new file mode 100644 index 0000000..5271e18 --- /dev/null +++ b/internal/client/output/output_client_interface.go @@ -0,0 +1,11 @@ +package output + +import ( + "io" +) + +type OutputClient interface { + GetWriter(path string) (io.Writer, error) + ReadMetadata(string) (map[string]string, error) + IsMissing(path string) (bool, error) +} diff --git a/internal/processor/processor_interface.go b/internal/processor/processor_interface.go new file mode 100644 index 0000000..8a8f803 --- /dev/null +++ b/internal/processor/processor_interface.go @@ -0,0 +1,8 @@ +package processor + +import "io" + +type Processor interface { + DeductOutputPath(inputPath string) string + Process(ext string, reader io.Reader, writer io.Writer) error +} diff --git a/internal/processor/webp.go b/internal/processor/webp.go new file mode 100644 index 0000000..984aab1 --- /dev/null +++ b/internal/processor/webp.go @@ -0,0 +1,81 @@ +package processor + +import ( + "fmt" + "image" + "image/jpeg" + "image/png" + "io" + "strings" + + "golang.org/x/image/draw" + + "github.com/SayaAndy/saya-today-thumbnail-generator/config" + "github.com/kolesa-team/go-webp/encoder" + "github.com/kolesa-team/go-webp/webp" +) + +var _ Processor = (*WebpProcessor)(nil) + +type WebpProcessor struct { + maxWidth int + maxHeight int + quality int +} + +func NewWebpProcessor(cfg *config.ProcessorConfig) (*WebpProcessor, error) { + if cfg.Type != "webp" { + return nil, fmt.Errorf("invalid storage type for WebpProcessor") + } + webpCfg := cfg.Config.(*config.WebpConfig) + + return &WebpProcessor{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality}, nil +} + +func (p *WebpProcessor) DeductOutputPath(inputPath string) string { + pathParts := strings.Split(inputPath, ".") + if len(pathParts) < 2 { + return inputPath + ".webp" + } + pathParts[len(pathParts)-1] = "webp" + return strings.Join(pathParts, ".") +} + +func (p *WebpProcessor) Process(contentType string, reader io.Reader, writer io.Writer) error { + var src image.Image + var err error + + switch contentType { + case "image/jpeg": + src, err = jpeg.Decode(reader) + if err != nil { + return fmt.Errorf("decode jpeg: %w", err) + } + case "image/png": + src, err = png.Decode(reader) + if err != nil { + return fmt.Errorf("decode png: %w", err) + } + } + + opts, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, float32(p.quality)) + if err != nil { + return fmt.Errorf("create webp encoder options: %w", err) + } + + xCoef := p.maxWidth / src.Bounds().Max.X + yCoef := p.maxHeight / src.Bounds().Max.Y + if xCoef > 1 && yCoef > 1 { + return webp.Encode(writer, src, opts) + } + + minCoef := xCoef + if yCoef < minCoef { + minCoef = yCoef + } + + dst := image.NewRGBA(image.Rect(0, 0, src.Bounds().Max.X*xCoef, src.Bounds().Max.Y*xCoef)) + draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil) + + return webp.Encode(writer, dst, opts) +} -- cgit v1.3.1+13