summaryrefslogtreecommitdiff
path: root/internal
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat (limited to 'internal')
-rw-r--r--internal/client/input/b2.go12
-rw-r--r--internal/client/input/input_client_interface.go10
-rw-r--r--internal/client/input/local_unix.go108
-rw-r--r--internal/client/input/s3.go157
-rw-r--r--internal/client/output/b2.go17
-rw-r--r--internal/client/output/local_unix.go129
-rw-r--r--internal/client/output/output_client_interface.go12
-rw-r--r--internal/client/output/s3.go170
-rw-r--r--internal/converter/converter_interface.go21
-rw-r--r--internal/converter/jpeg.go114
-rw-r--r--internal/converter/processor_interface.go8
-rw-r--r--internal/converter/webp.go79
12 files changed, 46 insertions, 791 deletions
diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go
index fe94333..54a6b09 100644
--- a/internal/client/input/b2.go
+++ b/internal/client/input/b2.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"slices"
+ "strconv"
"strings"
"github.com/Backblaze/blazer/b2"
@@ -16,12 +17,11 @@ var _ InputClient = (*B2InputClient)(nil)
type B2InputClient struct {
prefix string
bucket *b2.Bucket
- bucketName string
b2cl *b2.Client
knownExtensions []string
}
-func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) {
+func NewB2InputClient(cfg *config.InputConfig) (*B2InputClient, error) {
if cfg.Storage.Type != "b2" {
return nil, fmt.Errorf("invalid storage type for B2InputClient")
}
@@ -37,7 +37,7 @@ func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) {
return nil, err
}
- return &B2InputClient{b2cl: b2cl, bucket: bucket, bucketName: b2cfg.BucketName, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
+ return &B2InputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
}
func (c *B2InputClient) Scan() ([]string, error) {
@@ -102,8 +102,8 @@ func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
FirstCreated: attrs.UploadTimestamp,
LastModified: attrs.LastModified,
Misc: attrs.Info,
- Size: attrs.Size,
}
+ metadata.Misc["Size"] = strconv.FormatInt(attrs.Size, 10)
switch attrs.Status {
case b2.Uploaded:
@@ -121,10 +121,6 @@ func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
return &metadata, nil
}
-func (c *B2InputClient) ID(path string) string {
- return fmt.Sprintf("b2://%s/%s%s", c.bucketName, c.prefix, path)
-}
-
func (c *B2InputClient) GetReader(path string) (io.ReadCloser, error) {
obj := c.bucket.Object(c.prefix + path)
if obj == nil {
diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go
index dca64db..c896ca4 100644
--- a/internal/client/input/input_client_interface.go
+++ b/internal/client/input/input_client_interface.go
@@ -3,15 +3,12 @@ package input
import (
"io"
"time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
)
type InputClient interface {
Scan() ([]string, error)
ReadMetadata(string) (*MetadataStruct, error)
GetReader(string) (io.ReadCloser, error)
- ID(path string) string
}
type MetadataStruct struct {
@@ -21,12 +18,5 @@ type MetadataStruct struct {
ContentType string
FirstCreated time.Time
LastModified time.Time
- Size int64
Misc map[string]string
}
-
-var NewInputClientMap = map[string]func(cfg *config.InputConfig) (InputClient, error){
- "b2": NewB2InputClient,
- "s3": NewS3InputClient,
- "local-unix": NewLocalUnixInputClient,
-}
diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go
deleted file mode 100644
index 21b08a9..0000000
--- a/internal/client/input/local_unix.go
+++ /dev/null
@@ -1,108 +0,0 @@
-package input
-
-import (
- "fmt"
- "io"
- "log/slog"
- "mime"
- "os"
- "slices"
- "strconv"
- "strings"
- "syscall"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-var _ InputClient = (*LocalUnixInputClient)(nil)
-
-type LocalUnixInputClient struct {
- path string
- maxDepth int
- knownExtensions []string
-}
-
-func NewLocalUnixInputClient(cfg *config.InputConfig) (InputClient, error) {
- if cfg.Storage.Type != "local-unix" {
- return nil, fmt.Errorf("invalid storage type for LocalUnixInputClient")
- }
- localCfg := cfg.Storage.Config.(*config.InputLocalUnixConfig)
-
- return &LocalUnixInputClient{
- path: localCfg.Path,
- maxDepth: localCfg.MaxDepth,
- knownExtensions: cfg.KnownExtensions,
- }, nil
-}
-
-func (c *LocalUnixInputClient) Scan() ([]string, error) {
- return c.recursiveScan(c.path, c.maxDepth)
-}
-
-func (c *LocalUnixInputClient) recursiveScan(dir string, depth int) ([]string, error) {
- filePaths := make([]string, 0)
- entries, err := os.ReadDir(dir)
- if err != nil {
- return nil, fmt.Errorf("fail to read directory: %w", err)
- }
-
- for _, entry := range entries {
- if entry.IsDir() && depth > 0 {
- subFilePaths, err := c.recursiveScan(dir+entry.Name()+"/", depth-1)
- if err != nil {
- return nil, fmt.Errorf("fail to scan subdirectory '%s': %w", entry.Name(), err)
- }
- filePaths = append(filePaths, subFilePaths...)
- } else if !entry.IsDir() {
- nameParts := strings.Split(entry.Name(), ".")
- if len(nameParts) < 2 {
- continue
- }
- if slices.Contains(c.knownExtensions, strings.ToLower(nameParts[len(nameParts)-1])) {
- filePaths = append(filePaths, strings.TrimPrefix(dir+entry.Name(), c.path))
- }
- fmt.Println(entry.Name())
- }
- }
-
- return filePaths, nil
-}
-
-func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- nodePathParts := strings.Split(path, "/")
- nodeName := nodePathParts[len(nodePathParts)-1]
- nodeNameParts := strings.Split(nodeName, ".")
- nodeExt := ""
- if len(nodeNameParts) >= 2 {
- nodeExt = nodeNameParts[len(nodeNameParts)-1]
- }
- slog.Debug("got a file extension", slog.String("extension", nodeExt), slog.String("path", path), slog.String("filename", nodeName))
-
- fileInfo, err := os.Stat(c.path + path)
- if err != nil {
- return nil, fmt.Errorf("fail to read file info: %w", err)
- }
-
- stat_t := fileInfo.Sys().(*syscall.Stat_t)
- creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
-
- return &MetadataStruct{
- Name: fileInfo.Name(),
- StorageType: "local-unix",
- Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16),
- ContentType: mime.TypeByExtension("." + nodeExt),
- FirstCreated: creationTime,
- LastModified: fileInfo.ModTime(),
- Size: fileInfo.Size(),
- Misc: map[string]string{},
- }, nil
-}
-
-func (c *LocalUnixInputClient) ID(path string) string {
- return fmt.Sprintf("local-unix://%s%s", c.path, path)
-}
-
-func (c *LocalUnixInputClient) GetReader(path string) (io.ReadCloser, error) {
- return os.Open(c.path + path)
-}
diff --git a/internal/client/input/s3.go b/internal/client/input/s3.go
deleted file mode 100644
index 00684c2..0000000
--- a/internal/client/input/s3.go
+++ /dev/null
@@ -1,157 +0,0 @@
-package input
-
-import (
- "context"
- "fmt"
- "io"
- "slices"
- "strings"
-
- "github.com/aws/aws-sdk-go-v2/aws"
- awsconfig "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-var _ InputClient = (*S3InputClient)(nil)
-
-type S3InputClient struct {
- prefix string
- bucketName string
- s3cl *s3.Client
- knownExtensions []string
-}
-
-func NewS3InputClient(cfg *config.InputConfig) (InputClient, error) {
- if cfg.Storage.Type != "s3" {
- return nil, fmt.Errorf("invalid storage type for S3InputClient")
- }
- s3cfg := cfg.Storage.Config.(*config.S3Config)
-
- s3cl, err := newS3Client(s3cfg)
- if err != nil {
- return nil, fmt.Errorf("create S3 client: %w", err)
- }
-
- return &S3InputClient{
- s3cl: s3cl,
- bucketName: s3cfg.BucketName,
- prefix: s3cfg.Prefix,
- knownExtensions: cfg.KnownExtensions,
- }, nil
-}
-
-func (c *S3InputClient) Scan() ([]string, error) {
- var filePaths []string
-
- paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
- Bucket: aws.String(c.bucketName),
- Prefix: aws.String(c.prefix),
- })
-
- for paginator.HasMorePages() {
- output, err := paginator.NextPage(context.Background())
- if err != nil {
- return nil, fmt.Errorf("list S3 objects: %w", err)
- }
-
- for _, obj := range output.Contents {
- name := aws.ToString(obj.Key)
-
- if len(c.knownExtensions) != 0 {
- nameParts := strings.Split(name, ".")
- if len(nameParts) < 2 {
- continue
- }
- ext := strings.ToLower(nameParts[len(nameParts)-1])
- if !slices.Contains(c.knownExtensions, ext) {
- continue
- }
- }
-
- filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
- }
- }
-
- return filePaths, nil
-}
-
-func (c *S3InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- key := c.prefix + path
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("head S3 object %s: %w", key, err)
- }
-
- metadata := MetadataStruct{
- Name: key,
- StorageType: "s3",
- Hash: strings.Trim(aws.ToString(head.ETag), "\""),
- ContentType: aws.ToString(head.ContentType),
- Misc: head.Metadata,
- }
-
- if head.ContentLength != nil {
- metadata.Size = *head.ContentLength
- }
- if head.LastModified != nil {
- metadata.LastModified = *head.LastModified
- metadata.FirstCreated = *head.LastModified
- }
-
- return &metadata, nil
-}
-
-func (c *S3InputClient) ID(path string) string {
- return fmt.Sprintf("s3://%s/%s%s", c.bucketName, c.prefix, path)
-}
-
-func (c *S3InputClient) GetReader(path string) (io.ReadCloser, error) {
- key := c.prefix + path
-
- output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("get S3 object %s: %w", key, err)
- }
-
- return output.Body, nil
-}
-
-func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(cfg.Region),
- }
-
- if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
- opts = append(opts, awsconfig.WithCredentialsProvider(
- credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
- ))
- }
-
- awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
- if err != nil {
- return nil, fmt.Errorf("load AWS config: %w", err)
- }
-
- var s3Opts []func(*s3.Options)
- if cfg.Endpoint != "" {
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.BaseEndpoint = aws.String(cfg.Endpoint)
- })
- }
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.UsePathStyle = cfg.UsePathStyle
- o.DisableLogOutputChecksumValidationSkipped = true
- })
-
- return s3.NewFromConfig(awsCfg, s3Opts...), nil
-}
diff --git a/internal/client/output/b2.go b/internal/client/output/b2.go
index 0616368..6979e5b 100644
--- a/internal/client/output/b2.go
+++ b/internal/client/output/b2.go
@@ -2,10 +2,9 @@ package output
import (
"context"
- "encoding/json"
"fmt"
"io"
- "log/slog"
+ "strconv"
"github.com/Backblaze/blazer/b2"
"github.com/SayaAndy/saya-today-thumbnail-generator/config"
@@ -20,7 +19,7 @@ type B2OutputClient struct {
b2cl *b2.Client
}
-func NewB2OutputClient(cfg *config.OutputConfig) (OutputClient, error) {
+func NewB2OutputClient(cfg *config.OutputConfig) (*B2OutputClient, error) {
if cfg.Storage.Type != "b2" {
return nil, fmt.Errorf("invalid storage type for B2OutputClient")
}
@@ -39,7 +38,7 @@ func NewB2OutputClient(cfg *config.OutputConfig) (OutputClient, error) {
return &B2OutputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
}
-func (c *B2OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) {
+func (c *B2OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct) (io.WriteCloser, error) {
obj := c.bucket.Object(c.prefix + path)
if obj == nil {
return nil, fmt.Errorf("failed to reference object in B2 bucket")
@@ -47,7 +46,6 @@ func (c *B2OutputClient) GetWriter(path string, inputMetadata *input.MetadataStr
attrs := &b2.Attrs{Info: make(map[string]string)}
attrs.Info["sha1-original"] = inputMetadata.Hash
- attrs.ContentType = outputContentType
return obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs)), nil
}
@@ -72,8 +70,8 @@ func (c *B2OutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
FirstCreated: attrs.UploadTimestamp,
LastModified: attrs.LastModified,
Misc: attrs.Info,
- Size: attrs.Size,
}
+ metadata.Misc["Size"] = strconv.FormatInt(attrs.Size, 10)
switch attrs.Status {
case b2.Uploaded:
@@ -102,12 +100,5 @@ func (c *B2OutputClient) IsMissing(path string) bool {
return true
}
- attrsJson, _ := json.Marshal(attrs)
- slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(attrsJson)))
-
- if attrs.Size == 0 {
- return true
- }
-
return attrs.Status == b2.Hider
}
diff --git a/internal/client/output/local_unix.go b/internal/client/output/local_unix.go
deleted file mode 100644
index 6f9a7b3..0000000
--- a/internal/client/output/local_unix.go
+++ /dev/null
@@ -1,129 +0,0 @@
-package output
-
-import (
- "fmt"
- "io"
- "log/slog"
- "mime"
- "os"
- "strconv"
- "strings"
- "syscall"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "golang.org/x/sys/unix"
-)
-
-var _ OutputClient = (*LocalUnixOutputClient)(nil)
-
-type LocalUnixOutputClient struct {
- path string
- fileMode uint32
- dirMode uint32
- attrMode string
-}
-
-func NewLocalUnixOutputClient(cfg *config.OutputConfig) (OutputClient, error) {
- if cfg.Storage.Type != "local-unix" {
- return nil, fmt.Errorf("invalid storage type for LocalUnixOutputClient")
- }
- localCfg := cfg.Storage.Config.(*config.OutputLocalUnixConfig)
-
- fpm, err := strconv.ParseInt(localCfg.FilePermissionMode, 8, 32)
- if err != nil {
- return nil, fmt.Errorf("fail to parse file permission mode as an octal number: %w", err)
- }
-
- dpm, err := strconv.ParseInt(localCfg.DirPermissionMode, 8, 32)
- if err != nil {
- return nil, fmt.Errorf("fail to parse directory permission mode as an octal number: %w", err)
- }
-
- return &LocalUnixOutputClient{localCfg.Path, uint32(fpm), uint32(dpm), localCfg.AttributesImplementation}, nil
-}
-
-func (c *LocalUnixOutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, _ string) (io.WriteCloser, error) {
- pathSegments := strings.Split(path, "/")
- dirpath := strings.Join(pathSegments[0:len(pathSegments)-1], "/")
- if err := os.MkdirAll(c.path+dirpath, os.FileMode(c.dirMode)); err != nil {
- return nil, fmt.Errorf("fail to mkdir parent directories for a path: %w", err)
- }
-
- var err error
- if _, err = os.Stat(c.path + path); err != nil && !os.IsNotExist(err) {
- return nil, fmt.Errorf("fail to stat a file, although file exists: %w", err)
- }
- if os.IsNotExist(err) {
- f, err := os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE, os.FileMode(c.fileMode))
- if err != nil {
- return nil, fmt.Errorf("fail to create a file: %w", err)
- }
- f.Close()
- }
-
- switch c.attrMode {
- case "xattr":
- if err := unix.Setxattr(c.path+path, "user.originalfile.mddate", []byte(strconv.FormatInt(inputMetadata.LastModified.Unix(), 16)), 0); err != nil {
- return nil, fmt.Errorf("fail to write user.originalfile.mddate xattribute: %w", err)
- }
- case "none":
- default:
- return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode)
- }
-
- return os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(c.fileMode))
-}
-
-func (c *LocalUnixOutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- nodePathParts := strings.Split(path, "/")
- nodeName := nodePathParts[len(nodePathParts)-1]
- nodeNameParts := strings.Split(nodeName, ".")
- nodeExt := ""
- if len(nodeNameParts) >= 2 {
- nodeExt = nodeNameParts[len(nodeNameParts)-1]
- }
-
- fileInfo, err := os.Stat(c.path + path)
- if err != nil {
- return nil, fmt.Errorf("fail to read file info: %w", err)
- }
-
- stat_t := fileInfo.Sys().(*syscall.Stat_t)
- creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
-
- mddateOriginal := make([]byte, 0)
- switch c.attrMode {
- case "xattr":
- sz, err := unix.Getxattr(c.path+path, "user.originalfile.mddate", nil)
- if err != nil {
- slog.Warn("fail to get size of user.originalfile.mddate attribute, proceeding as if no such attribute is there", slog.String("error", err.Error()))
- break
- }
- mddateOriginal = make([]byte, sz)
- if _, err = unix.Getxattr(c.path+path, "user.originalfile.mddate", mddateOriginal); err != nil {
- return nil, fmt.Errorf("fail to get user.originalfile.mddate attribute: %w", err)
- }
- case "none":
- default:
- return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode)
- }
-
- return &MetadataStruct{
- Name: fileInfo.Name(),
- StorageType: "local-unix",
- Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16),
- HashOriginal: string(mddateOriginal),
- ContentType: mime.TypeByExtension("." + nodeExt),
- FirstCreated: creationTime,
- LastModified: fileInfo.ModTime(),
- Size: fileInfo.Size(),
- Misc: map[string]string{},
- }, nil
-}
-
-func (c *LocalUnixOutputClient) IsMissing(path string) bool {
- _, err := os.Stat(c.path + path)
- return err != nil
-}
diff --git a/internal/client/output/output_client_interface.go b/internal/client/output/output_client_interface.go
index 4415aa4..4ec4728 100644
--- a/internal/client/output/output_client_interface.go
+++ b/internal/client/output/output_client_interface.go
@@ -4,13 +4,12 @@ import (
"io"
"time"
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
"github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
)
type OutputClient interface {
- GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error)
- ReadMetadata(path string) (*MetadataStruct, error)
+ GetWriter(path string, inputMetadata *input.MetadataStruct) (io.WriteCloser, error)
+ ReadMetadata(string) (*MetadataStruct, error)
IsMissing(path string) bool
}
@@ -22,12 +21,5 @@ type MetadataStruct struct {
ContentType string
FirstCreated time.Time
LastModified time.Time
- Size int64
Misc map[string]string
}
-
-var NewOutputClientMap = map[string]func(cfg *config.OutputConfig) (OutputClient, error){
- "b2": NewB2OutputClient,
- "s3": NewS3OutputClient,
- "local-unix": NewLocalUnixOutputClient,
-}
diff --git a/internal/client/output/s3.go b/internal/client/output/s3.go
deleted file mode 100644
index f0527fb..0000000
--- a/internal/client/output/s3.go
+++ /dev/null
@@ -1,170 +0,0 @@
-package output
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "fmt"
- "io"
- "log/slog"
- "strings"
-
- "github.com/aws/aws-sdk-go-v2/aws"
- awsconfig "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
-)
-
-var _ OutputClient = (*S3OutputClient)(nil)
-
-type S3OutputClient struct {
- prefix string
- bucketName string
- s3cl *s3.Client
-}
-
-func NewS3OutputClient(cfg *config.OutputConfig) (OutputClient, error) {
- if cfg.Storage.Type != "s3" {
- return nil, fmt.Errorf("invalid storage type for S3OutputClient")
- }
- s3cfg := cfg.Storage.Config.(*config.S3Config)
-
- s3cl, err := newS3Client(s3cfg)
- if err != nil {
- return nil, fmt.Errorf("create S3 client: %w", err)
- }
-
- return &S3OutputClient{
- s3cl: s3cl,
- bucketName: s3cfg.BucketName,
- prefix: s3cfg.Prefix,
- }, nil
-}
-
-func (c *S3OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) {
- key := c.prefix + path
-
- return &s3WriteCloser{
- key: key,
- bucketName: c.bucketName,
- s3cl: c.s3cl,
- contentType: outputContentType,
- hashOriginal: inputMetadata.Hash,
- buf: &bytes.Buffer{},
- }, nil
-}
-
-func (c *S3OutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- key := c.prefix + path
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("head S3 object %s: %w", key, err)
- }
-
- metadata := MetadataStruct{
- Name: key,
- StorageType: "s3",
- Hash: strings.Trim(aws.ToString(head.ETag), "\""),
- HashOriginal: head.Metadata["sha1-original"],
- ContentType: aws.ToString(head.ContentType),
- Misc: head.Metadata,
- }
-
- if head.ContentLength != nil {
- metadata.Size = *head.ContentLength
- }
- if head.LastModified != nil {
- metadata.LastModified = *head.LastModified
- metadata.FirstCreated = *head.LastModified
- }
-
- return &metadata, nil
-}
-
-func (c *S3OutputClient) IsMissing(path string) bool {
- key := c.prefix + path
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return true
- }
-
- headJson, _ := json.Marshal(head)
- slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(headJson)))
-
- if head.ContentLength == nil || *head.ContentLength == 0 {
- return true
- }
-
- return false
-}
-
-// s3WriteCloser buffers writes and uploads to S3 on Close.
-type s3WriteCloser struct {
- key string
- bucketName string
- s3cl *s3.Client
- contentType string
- hashOriginal string
- buf *bytes.Buffer
-}
-
-func (w *s3WriteCloser) Write(p []byte) (int, error) {
- return w.buf.Write(p)
-}
-
-func (w *s3WriteCloser) Close() error {
- _, err := w.s3cl.PutObject(context.Background(), &s3.PutObjectInput{
- Bucket: aws.String(w.bucketName),
- Key: aws.String(w.key),
- Body: bytes.NewReader(w.buf.Bytes()),
- ContentType: aws.String(w.contentType),
- Metadata: map[string]string{
- "sha1-original": w.hashOriginal,
- },
- })
- if err != nil {
- return fmt.Errorf("put S3 object %s: %w", w.key, err)
- }
- return nil
-}
-
-func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(cfg.Region),
- }
-
- if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
- opts = append(opts, awsconfig.WithCredentialsProvider(
- credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
- ))
- }
-
- awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
- if err != nil {
- return nil, fmt.Errorf("load AWS config: %w", err)
- }
-
- var s3Opts []func(*s3.Options)
- if cfg.Endpoint != "" {
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.BaseEndpoint = aws.String(cfg.Endpoint)
- })
- }
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.UsePathStyle = cfg.UsePathStyle
- o.DisableLogOutputChecksumValidationSkipped = true
- })
-
- return s3.NewFromConfig(awsCfg, s3Opts...), nil
-}
diff --git a/internal/converter/converter_interface.go b/internal/converter/converter_interface.go
deleted file mode 100644
index abff99d..0000000
--- a/internal/converter/converter_interface.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package converter
-
-import (
- "io"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output"
-)
-
-type Converter interface {
- Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error
- DeductOutputPath(inputPath string) string
- ReadMetadata(path string) (*output.MetadataStruct, error)
- IsMissing(path string) bool
-}
-
-var NewConverterMap = map[string]func(cfg *config.ConverterConfig) (Converter, error){
- "webp": NewWebpConverter,
- "jpeg": NewJpegConverter,
-}
diff --git a/internal/converter/jpeg.go b/internal/converter/jpeg.go
deleted file mode 100644
index a35a47c..0000000
--- a/internal/converter/jpeg.go
+++ /dev/null
@@ -1,114 +0,0 @@
-package converter
-
-import (
- "fmt"
- "image"
- "image/jpeg"
- "image/png"
- "io"
- "log/slog"
- "path/filepath"
- "strings"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output"
- "github.com/kolesa-team/go-webp/webp"
- "golang.org/x/image/draw"
-)
-
-var _ Converter = (*JpegConverter)(nil)
-
-type JpegConverter struct {
- maxWidth int
- maxHeight int
- extensionName string
- quality int
- outputClient output.OutputClient
-}
-
-func NewJpegConverter(cfg *config.ConverterConfig) (Converter, error) {
- if cfg.Type != "jpeg" {
- return nil, fmt.Errorf("invalid storage type for JpegConverter")
- }
- jpegCfg := cfg.Config.(*config.JpegConfig)
-
- outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output)
- if err != nil {
- return nil, fmt.Errorf("fail to initialize output client: %w", err)
- }
-
- extensionName := ".jpg"
- if jpegCfg.ExtensionName != "" {
- extensionName = "." + jpegCfg.ExtensionName
- }
-
- return &JpegConverter{jpegCfg.Size.MaxWidth, jpegCfg.Size.MaxHeight, extensionName, jpegCfg.Quality, outputClient}, nil
-}
-
-func (p *JpegConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error {
- var src image.Image
-
- writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/jpeg")
- if err != nil {
- return fmt.Errorf("fail to initialize writer for output: %w", err)
- }
- defer writer.Close()
-
- switch inputMetadata.ContentType {
- case "image/jpeg":
- src, err = jpeg.Decode(reader)
- if err != nil {
- return fmt.Errorf("decode jpeg: %w", err)
- }
- case "image/png":
- src, err = png.Decode(reader)
- if err != nil {
- return fmt.Errorf("decode png: %w", err)
- }
- case "image/webp":
- src, err = webp.Decode(reader, nil)
- if err != nil {
- return fmt.Errorf("decode webp: %w", err)
- }
- default:
- return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType)
- }
-
- xCoef := 1.0
- if p.maxWidth > 0 {
- xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X)
- }
- yCoef := 1.0
- if p.maxHeight > 0 {
- yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y)
- }
- slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef))
-
- minCoef := xCoef
- if yCoef < minCoef {
- minCoef = yCoef
- }
-
- if minCoef >= 1.0 {
- return jpeg.Encode(writer, src, &jpeg.Options{Quality: p.quality})
- }
-
- dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5)))
- draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil)
-
- return jpeg.Encode(writer, dst, &jpeg.Options{Quality: p.quality})
-}
-
-func (p *JpegConverter) DeductOutputPath(inputPath string) string {
- withoutExt, _ := strings.CutSuffix(inputPath, filepath.Ext(inputPath))
- return withoutExt + p.extensionName
-}
-
-func (p *JpegConverter) ReadMetadata(path string) (*output.MetadataStruct, error) {
- return p.outputClient.ReadMetadata(path)
-}
-
-func (p *JpegConverter) IsMissing(path string) bool {
- return p.outputClient.IsMissing(path)
-}
diff --git a/internal/converter/processor_interface.go b/internal/converter/processor_interface.go
new file mode 100644
index 0000000..88b5ac3
--- /dev/null
+++ b/internal/converter/processor_interface.go
@@ -0,0 +1,8 @@
+package converter
+
+import "io"
+
+type Converter interface {
+ DeductOutputPath(inputPath string) string
+ Process(ext string, reader io.ReadCloser, writer io.WriteCloser) error
+}
diff --git a/internal/converter/webp.go b/internal/converter/webp.go
index fbffd01..b7f1216 100644
--- a/internal/converter/webp.go
+++ b/internal/converter/webp.go
@@ -12,8 +12,6 @@ import (
"golang.org/x/image/draw"
"github.com/SayaAndy/saya-today-thumbnail-generator/config"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
- "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output"
"github.com/kolesa-team/go-webp/encoder"
"github.com/kolesa-team/go-webp/webp"
)
@@ -21,36 +19,37 @@ import (
var _ Converter = (*WebpConverter)(nil)
type WebpConverter struct {
- maxWidth int
- maxHeight int
- quality int
- outputClient output.OutputClient
+ maxWidth int
+ maxHeight int
+ quality int
}
-func NewWebpConverter(cfg *config.ConverterConfig) (Converter, error) {
+func NewWebpConverter(cfg *config.ConverterConfig) (*WebpConverter, error) {
if cfg.Type != "webp" {
return nil, fmt.Errorf("invalid storage type for WebpConverter")
}
webpCfg := cfg.Config.(*config.WebpConfig)
- outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output)
- if err != nil {
- return nil, fmt.Errorf("fail to initialize output client: %w", err)
- }
+ return &WebpConverter{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality}, nil
+}
- return &WebpConverter{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality, outputClient}, nil
+func (p *WebpConverter) DeductOutputPath(inputPath string) string {
+ pathParts := strings.Split(inputPath, ".")
+ if len(pathParts) < 2 {
+ return inputPath + ".webp"
+ }
+ pathParts[len(pathParts)-1] = "webp"
+ return strings.Join(pathParts, ".")
}
-func (p *WebpConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error {
+func (p *WebpConverter) Process(contentType string, reader io.ReadCloser, writer io.WriteCloser) error {
var src image.Image
+ var err error
- writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/webp")
- if err != nil {
- return fmt.Errorf("fail to initialize writer for output: %w", err)
- }
+ defer reader.Close()
defer writer.Close()
- switch inputMetadata.ContentType {
+ switch contentType {
case "image/jpeg":
src, err = jpeg.Decode(reader)
if err != nil {
@@ -61,13 +60,8 @@ func (p *WebpConverter) Process(inputMetadata *input.MetadataStruct, reader io.R
if err != nil {
return fmt.Errorf("decode png: %w", err)
}
- case "image/webp":
- src, err = webp.Decode(reader, nil)
- if err != nil {
- return fmt.Errorf("decode webp: %w", err)
- }
default:
- return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType)
+ return fmt.Errorf("unsupported content type: %s", contentType)
}
opts, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, float32(p.quality))
@@ -75,44 +69,27 @@ func (p *WebpConverter) Process(inputMetadata *input.MetadataStruct, reader io.R
return fmt.Errorf("create webp encoder options: %w", err)
}
- xCoef := 1.0
- if p.maxWidth > 0 {
- xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X)
+ xCoef := float64(p.maxWidth) / float64(src.Bounds().Max.X)
+ if p.maxWidth == 0 {
+ xCoef = 1
}
- yCoef := 1.0
- if p.maxHeight > 0 {
- yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y)
+ yCoef := float64(p.maxHeight) / float64(src.Bounds().Max.Y)
+ if p.maxHeight == 0 {
+ yCoef = 1
}
slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef))
+ if xCoef > 1 && yCoef > 1 {
+ return webp.Encode(writer, src, opts)
+ }
+
minCoef := xCoef
if yCoef < minCoef {
minCoef = yCoef
}
- if minCoef >= 1.0 {
- return webp.Encode(writer, src, opts)
- }
-
dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5)))
draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil)
return webp.Encode(writer, dst, opts)
}
-
-func (p *WebpConverter) DeductOutputPath(inputPath string) string {
- pathParts := strings.Split(inputPath, ".")
- if len(pathParts) < 2 {
- return inputPath + ".webp"
- }
- pathParts[len(pathParts)-1] = "webp"
- return strings.Join(pathParts, ".")
-}
-
-func (p *WebpConverter) ReadMetadata(path string) (*output.MetadataStruct, error) {
- return p.outputClient.ReadMetadata(path)
-}
-
-func (p *WebpConverter) IsMissing(path string) bool {
- return p.outputClient.IsMissing(path)
-}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-21 10:44:39 +0000