Diffstat (limited to 'internal')
| -rw-r--r-- | internal/client/input/b2.go | 135 | ||||
| -rw-r--r-- | internal/client/input/input_client_interface.go | 32 | ||||
| -rw-r--r-- | internal/client/input/local_unix.go | 108 | ||||
| -rw-r--r-- | internal/client/input/s3.go | 157 | ||||
| -rw-r--r-- | internal/client/output/b2.go | 113 | ||||
| -rw-r--r-- | internal/client/output/local_unix.go | 129 | ||||
| -rw-r--r-- | internal/client/output/output_client_interface.go | 33 | ||||
| -rw-r--r-- | internal/client/output/s3.go | 170 | ||||
| -rw-r--r-- | internal/converter/converter_interface.go | 21 | ||||
| -rw-r--r-- | internal/converter/jpeg.go | 114 | ||||
| -rw-r--r-- | internal/converter/webp.go | 118 |
11 files changed, 0 insertions, 1130 deletions
diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go deleted file mode 100644 index fe94333..0000000 --- a/internal/client/input/b2.go +++ /dev/null @@ -1,135 +0,0 @@ -package input - -import ( - "context" - "fmt" - "io" - "slices" - "strings" - - "github.com/Backblaze/blazer/b2" - "github.com/SayaAndy/saya-today-thumbnail-generator/config" -) - -var _ InputClient = (*B2InputClient)(nil) - -type B2InputClient struct { - prefix string - bucket *b2.Bucket - bucketName string - b2cl *b2.Client - knownExtensions []string -} - -func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) { - if cfg.Storage.Type != "b2" { - return nil, fmt.Errorf("invalid storage type for B2InputClient") - } - b2cfg := cfg.Storage.Config.(*config.B2Config) - - b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) - if err != nil { - return nil, err - } - - bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) - if err != nil { - return nil, err - } - - return &B2InputClient{b2cl: b2cl, bucket: bucket, bucketName: b2cfg.BucketName, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil -} - -func (c *B2InputClient) Scan() ([]string, error) { - filePaths := []string{} - - iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix)) - - for iter.Next() { - obj := iter.Object() - if obj == nil { - return nil, fmt.Errorf("failed to reference object in B2 bucket") - } - - attrs, err := obj.Attrs(context.Background()) - if err != nil { - return nil, fmt.Errorf("get attributes for object: %w", err) - } - - if attrs.Status != b2.Uploaded { - continue - } - - name := obj.Name() - - if len(c.knownExtensions) != 0 { - nameParts := strings.Split(name, ".") - if len(nameParts) < 2 { - continue - } - ext := strings.ToLower(nameParts[len(nameParts)-1]) - if !slices.Contains(c.knownExtensions, ext) { - continue - } - } - - filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix)) - } - - if err := iter.Err(); err != nil { - return nil, fmt.Errorf("iterate over B2 objects: %w", err) - } - - return filePaths, nil -} - -func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) { - obj := c.bucket.Object(c.prefix + path) - if obj == nil { - return nil, fmt.Errorf("object not found in B2 bucket") - } - - attrs, err := obj.Attrs(context.Background()) - if err != nil { - return nil, fmt.Errorf("get attributes for object: %w", err) - } - - metadata := MetadataStruct{ - Name: attrs.Name, - StorageType: "b2", - Hash: attrs.SHA1, - ContentType: attrs.ContentType, - FirstCreated: attrs.UploadTimestamp, - LastModified: attrs.LastModified, - Misc: attrs.Info, - Size: attrs.Size, - } - - switch attrs.Status { - case b2.Uploaded: - metadata.Misc["b2-status"] = "Uploaded" - case b2.Folder: - metadata.Misc["b2-status"] = "Folder" - case b2.Hider: - metadata.Misc["b2-status"] = "Hider" - case b2.Started: - metadata.Misc["b2-status"] = "Started" - default: - metadata.Misc["b2-status"] = "Unknown" - } - - return &metadata, nil -} - -func (c *B2InputClient) ID(path string) string { - return fmt.Sprintf("b2://%s/%s%s", c.bucketName, c.prefix, path) -} - -func (c *B2InputClient) GetReader(path string) (io.ReadCloser, error) { - obj := c.bucket.Object(c.prefix + path) - if obj == nil { - return nil, fmt.Errorf("failed to reference object in B2 bucket") - } - - return obj.NewReader(context.Background()), nil -} diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go deleted file mode 100644 index dca64db..0000000 --- a/internal/client/input/input_client_interface.go +++ /dev/null @@ -1,32 +0,0 @@ -package input - -import ( - "io" - "time" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" -) - -type InputClient interface { - Scan() ([]string, error) - ReadMetadata(string) (*MetadataStruct, error) - GetReader(string) (io.ReadCloser, error) - ID(path string) string -} - -type MetadataStruct struct { - Name string - StorageType string - Hash string - ContentType string - FirstCreated time.Time - LastModified time.Time - Size int64 - Misc map[string]string -} - -var NewInputClientMap = map[string]func(cfg *config.InputConfig) (InputClient, error){ - "b2": NewB2InputClient, - "s3": NewS3InputClient, - "local-unix": NewLocalUnixInputClient, -} diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go deleted file mode 100644 index 21b08a9..0000000 --- a/internal/client/input/local_unix.go +++ /dev/null @@ -1,108 +0,0 @@ -package input - -import ( - "fmt" - "io" - "log/slog" - "mime" - "os" - "slices" - "strconv" - "strings" - "syscall" - "time" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" -) - -var _ InputClient = (*LocalUnixInputClient)(nil) - -type LocalUnixInputClient struct { - path string - maxDepth int - knownExtensions []string -} - -func NewLocalUnixInputClient(cfg *config.InputConfig) (InputClient, error) { - if cfg.Storage.Type != "local-unix" { - return nil, fmt.Errorf("invalid storage type for LocalUnixInputClient") - } - localCfg := cfg.Storage.Config.(*config.InputLocalUnixConfig) - - return &LocalUnixInputClient{ - path: localCfg.Path, - maxDepth: localCfg.MaxDepth, - knownExtensions: cfg.KnownExtensions, - }, nil -} - -func (c *LocalUnixInputClient) Scan() ([]string, error) { - return c.recursiveScan(c.path, c.maxDepth) -} - -func (c *LocalUnixInputClient) recursiveScan(dir string, depth int) ([]string, error) { - filePaths := make([]string, 0) - entries, err := os.ReadDir(dir) - if err != nil { - return nil, fmt.Errorf("fail to read directory: %w", err) - } - - for _, entry := range entries { - if entry.IsDir() && depth > 0 { - subFilePaths, err := c.recursiveScan(dir+entry.Name()+"/", depth-1) - if err != nil { - return nil, fmt.Errorf("fail to scan subdirectory '%s': %w", entry.Name(), err) - } - filePaths = append(filePaths, subFilePaths...) - } else if !entry.IsDir() { - nameParts := strings.Split(entry.Name(), ".") - if len(nameParts) < 2 { - continue - } - if slices.Contains(c.knownExtensions, strings.ToLower(nameParts[len(nameParts)-1])) { - filePaths = append(filePaths, strings.TrimPrefix(dir+entry.Name(), c.path)) - } - fmt.Println(entry.Name()) - } - } - - return filePaths, nil -} - -func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error) { - nodePathParts := strings.Split(path, "/") - nodeName := nodePathParts[len(nodePathParts)-1] - nodeNameParts := strings.Split(nodeName, ".") - nodeExt := "" - if len(nodeNameParts) >= 2 { - nodeExt = nodeNameParts[len(nodeNameParts)-1] - } - slog.Debug("got a file extension", slog.String("extension", nodeExt), slog.String("path", path), slog.String("filename", nodeName)) - - fileInfo, err := os.Stat(c.path + path) - if err != nil { - return nil, fmt.Errorf("fail to read file info: %w", err) - } - - stat_t := fileInfo.Sys().(*syscall.Stat_t) - creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec) - - return &MetadataStruct{ - Name: fileInfo.Name(), - StorageType: "local-unix", - Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16), - ContentType: mime.TypeByExtension("." + nodeExt), - FirstCreated: creationTime, - LastModified: fileInfo.ModTime(), - Size: fileInfo.Size(), - Misc: map[string]string{}, - }, nil -} - -func (c *LocalUnixInputClient) ID(path string) string { - return fmt.Sprintf("local-unix://%s%s", c.path, path) -} - -func (c *LocalUnixInputClient) GetReader(path string) (io.ReadCloser, error) { - return os.Open(c.path + path) -} diff --git a/internal/client/input/s3.go b/internal/client/input/s3.go deleted file mode 100644 index 00684c2..0000000 --- a/internal/client/input/s3.go +++ /dev/null @@ -1,157 +0,0 @@ -package input - -import ( - "context" - "fmt" - "io" - "slices" - "strings" - - "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" -) - -var _ InputClient = (*S3InputClient)(nil) - -type S3InputClient struct { - prefix string - bucketName string - s3cl *s3.Client - knownExtensions []string -} - -func NewS3InputClient(cfg *config.InputConfig) (InputClient, error) { - if cfg.Storage.Type != "s3" { - return nil, fmt.Errorf("invalid storage type for S3InputClient") - } - s3cfg := cfg.Storage.Config.(*config.S3Config) - - s3cl, err := newS3Client(s3cfg) - if err != nil { - return nil, fmt.Errorf("create S3 client: %w", err) - } - - return &S3InputClient{ - s3cl: s3cl, - bucketName: s3cfg.BucketName, - prefix: s3cfg.Prefix, - knownExtensions: cfg.KnownExtensions, - }, nil -} - -func (c *S3InputClient) Scan() ([]string, error) { - var filePaths []string - - paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{ - Bucket: aws.String(c.bucketName), - Prefix: aws.String(c.prefix), - }) - - for paginator.HasMorePages() { - output, err := paginator.NextPage(context.Background()) - if err != nil { - return nil, fmt.Errorf("list S3 objects: %w", err) - } - - for _, obj := range output.Contents { - name := aws.ToString(obj.Key) - - if len(c.knownExtensions) != 0 { - nameParts := strings.Split(name, ".") - if len(nameParts) < 2 { - continue - } - ext := strings.ToLower(nameParts[len(nameParts)-1]) - if !slices.Contains(c.knownExtensions, ext) { - continue - } - } - - filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix)) - } - } - - return filePaths, nil -} - -func (c *S3InputClient) ReadMetadata(path string) (*MetadataStruct, error) { - key := c.prefix + path - - head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ - Bucket: aws.String(c.bucketName), - Key: aws.String(key), - }) - if err != nil { - return nil, fmt.Errorf("head S3 object %s: %w", key, err) - } - - metadata := MetadataStruct{ - Name: key, - StorageType: "s3", - Hash: strings.Trim(aws.ToString(head.ETag), "\""), - ContentType: aws.ToString(head.ContentType), - Misc: head.Metadata, - } - - if head.ContentLength != nil { - metadata.Size = *head.ContentLength - } - if head.LastModified != nil { - metadata.LastModified = *head.LastModified - metadata.FirstCreated = *head.LastModified - } - - return &metadata, nil -} - -func (c *S3InputClient) ID(path string) string { - return fmt.Sprintf("s3://%s/%s%s", c.bucketName, c.prefix, path) -} - -func (c *S3InputClient) GetReader(path string) (io.ReadCloser, error) { - key := c.prefix + path - - output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{ - Bucket: aws.String(c.bucketName), - Key: aws.String(key), - }) - if err != nil { - return nil, fmt.Errorf("get S3 object %s: %w", key, err) - } - - return output.Body, nil -} - -func newS3Client(cfg *config.S3Config) (*s3.Client, error) { - opts := []func(*awsconfig.LoadOptions) error{ - awsconfig.WithRegion(cfg.Region), - } - - if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" { - opts = append(opts, awsconfig.WithCredentialsProvider( - credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""), - )) - } - - awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) - if err != nil { - return nil, fmt.Errorf("load AWS config: %w", err) - } - - var s3Opts []func(*s3.Options) - if cfg.Endpoint != "" { - s3Opts = append(s3Opts, func(o *s3.Options) { - o.BaseEndpoint = aws.String(cfg.Endpoint) - }) - } - s3Opts = append(s3Opts, func(o *s3.Options) { - o.UsePathStyle = cfg.UsePathStyle - o.DisableLogOutputChecksumValidationSkipped = true - }) - - return s3.NewFromConfig(awsCfg, s3Opts...), nil -} diff --git a/internal/client/output/b2.go b/internal/client/output/b2.go deleted file mode 100644 index 0616368..0000000 --- a/internal/client/output/b2.go +++ /dev/null @@ -1,113 +0,0 @@ -package output - -import ( - "context" - "encoding/json" - "fmt" - "io" - "log/slog" - - "github.com/Backblaze/blazer/b2" - "github.com/SayaAndy/saya-today-thumbnail-generator/config" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" -) - -var _ OutputClient = (*B2OutputClient)(nil) - -type B2OutputClient struct { - prefix string - bucket *b2.Bucket - b2cl *b2.Client -} - -func NewB2OutputClient(cfg *config.OutputConfig) (OutputClient, error) { - if cfg.Storage.Type != "b2" { - return nil, fmt.Errorf("invalid storage type for B2OutputClient") - } - b2cfg := cfg.Storage.Config.(*config.B2Config) - - b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) - if err != nil { - return nil, err - } - - bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) - if err != nil { - return nil, err - } - - return &B2OutputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil -} - -func (c *B2OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) { - obj := c.bucket.Object(c.prefix + path) - if obj == nil { - return nil, fmt.Errorf("failed to reference object in B2 bucket") - } - - attrs := &b2.Attrs{Info: make(map[string]string)} - attrs.Info["sha1-original"] = inputMetadata.Hash - attrs.ContentType = outputContentType - - return obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs)), nil -} - -func (c *B2OutputClient) ReadMetadata(path string) (*MetadataStruct, error) { - obj := c.bucket.Object(c.prefix + path) - if obj == nil { - return nil, fmt.Errorf("failed to reference object in B2 bucket") - } - - attrs, err := obj.Attrs(context.Background()) - if err != nil { - return nil, fmt.Errorf("get attributes for object: %w", err) - } - - metadata := MetadataStruct{ - Name: attrs.Name, - StorageType: "b2", - Hash: attrs.SHA1, - HashOriginal: attrs.Info["sha1-original"], - ContentType: attrs.ContentType, - FirstCreated: attrs.UploadTimestamp, - LastModified: attrs.LastModified, - Misc: attrs.Info, - Size: attrs.Size, - } - - switch attrs.Status { - case b2.Uploaded: - metadata.Misc["b2-status"] = "Uploaded" - case b2.Folder: - metadata.Misc["b2-status"] = "Folder" - case b2.Hider: - metadata.Misc["b2-status"] = "Hider" - case b2.Started: - metadata.Misc["b2-status"] = "Started" - default: - metadata.Misc["b2-status"] = "Unknown" - } - - return &metadata, nil -} - -func (c *B2OutputClient) IsMissing(path string) bool { - obj := c.bucket.Object(c.prefix + path) - if obj == nil { - return true - } - - attrs, err := obj.Attrs(context.Background()) - if err != nil { - return true - } - - attrsJson, _ := json.Marshal(attrs) - slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(attrsJson))) - - if attrs.Size == 0 { - return true - } - - return attrs.Status == b2.Hider -} diff --git a/internal/client/output/local_unix.go b/internal/client/output/local_unix.go deleted file mode 100644 index 6f9a7b3..0000000 --- a/internal/client/output/local_unix.go +++ /dev/null @@ -1,129 +0,0 @@ -package output - -import ( - "fmt" - "io" - "log/slog" - "mime" - "os" - "strconv" - "strings" - "syscall" - "time" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" - "golang.org/x/sys/unix" -) - -var _ OutputClient = (*LocalUnixOutputClient)(nil) - -type LocalUnixOutputClient struct { - path string - fileMode uint32 - dirMode uint32 - attrMode string -} - -func NewLocalUnixOutputClient(cfg *config.OutputConfig) (OutputClient, error) { - if cfg.Storage.Type != "local-unix" { - return nil, fmt.Errorf("invalid storage type for LocalUnixOutputClient") - } - localCfg := cfg.Storage.Config.(*config.OutputLocalUnixConfig) - - fpm, err := strconv.ParseInt(localCfg.FilePermissionMode, 8, 32) - if err != nil { - return nil, fmt.Errorf("fail to parse file permission mode as an octal number: %w", err) - } - - dpm, err := strconv.ParseInt(localCfg.DirPermissionMode, 8, 32) - if err != nil { - return nil, fmt.Errorf("fail to parse directory permission mode as an octal number: %w", err) - } - - return &LocalUnixOutputClient{localCfg.Path, uint32(fpm), uint32(dpm), localCfg.AttributesImplementation}, nil -} - -func (c *LocalUnixOutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, _ string) (io.WriteCloser, error) { - pathSegments := strings.Split(path, "/") - dirpath := strings.Join(pathSegments[0:len(pathSegments)-1], "/") - if err := os.MkdirAll(c.path+dirpath, os.FileMode(c.dirMode)); err != nil { - return nil, fmt.Errorf("fail to mkdir parent directories for a path: %w", err) - } - - var err error - if _, err = os.Stat(c.path + path); err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("fail to stat a file, although file exists: %w", err) - } - if os.IsNotExist(err) { - f, err := os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE, os.FileMode(c.fileMode)) - if err != nil { - return nil, fmt.Errorf("fail to create a file: %w", err) - } - f.Close() - } - - switch c.attrMode { - case "xattr": - if err := unix.Setxattr(c.path+path, "user.originalfile.mddate", []byte(strconv.FormatInt(inputMetadata.LastModified.Unix(), 16)), 0); err != nil { - return nil, fmt.Errorf("fail to write user.originalfile.mddate xattribute: %w", err) - } - case "none": - default: - return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode) - } - - return os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(c.fileMode)) -} - -func (c *LocalUnixOutputClient) ReadMetadata(path string) (*MetadataStruct, error) { - nodePathParts := strings.Split(path, "/") - nodeName := nodePathParts[len(nodePathParts)-1] - nodeNameParts := strings.Split(nodeName, ".") - nodeExt := "" - if len(nodeNameParts) >= 2 { - nodeExt = nodeNameParts[len(nodeNameParts)-1] - } - - fileInfo, err := os.Stat(c.path + path) - if err != nil { - return nil, fmt.Errorf("fail to read file info: %w", err) - } - - stat_t := fileInfo.Sys().(*syscall.Stat_t) - creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec) - - mddateOriginal := make([]byte, 0) - switch c.attrMode { - case "xattr": - sz, err := unix.Getxattr(c.path+path, "user.originalfile.mddate", nil) - if err != nil { - slog.Warn("fail to get size of user.originalfile.mddate attribute, proceeding as if no such attribute is there", slog.String("error", err.Error())) - break - } - mddateOriginal = make([]byte, sz) - if _, err = unix.Getxattr(c.path+path, "user.originalfile.mddate", mddateOriginal); err != nil { - return nil, fmt.Errorf("fail to get user.originalfile.mddate attribute: %w", err) - } - case "none": - default: - return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode) - } - - return &MetadataStruct{ - Name: fileInfo.Name(), - StorageType: "local-unix", - Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16), - HashOriginal: string(mddateOriginal), - ContentType: mime.TypeByExtension("." + nodeExt), - FirstCreated: creationTime, - LastModified: fileInfo.ModTime(), - Size: fileInfo.Size(), - Misc: map[string]string{}, - }, nil -} - -func (c *LocalUnixOutputClient) IsMissing(path string) bool { - _, err := os.Stat(c.path + path) - return err != nil -} diff --git a/internal/client/output/output_client_interface.go b/internal/client/output/output_client_interface.go deleted file mode 100644 index 4415aa4..0000000 --- a/internal/client/output/output_client_interface.go +++ /dev/null @@ -1,33 +0,0 @@ -package output - -import ( - "io" - "time" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" -) - -type OutputClient interface { - GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) - ReadMetadata(path string) (*MetadataStruct, error) - IsMissing(path string) bool -} - -type MetadataStruct struct { - Name string - StorageType string - Hash string - HashOriginal string - ContentType string - FirstCreated time.Time - LastModified time.Time - Size int64 - Misc map[string]string -} - -var NewOutputClientMap = map[string]func(cfg *config.OutputConfig) (OutputClient, error){ - "b2": NewB2OutputClient, - "s3": NewS3OutputClient, - "local-unix": NewLocalUnixOutputClient, -} diff --git a/internal/client/output/s3.go b/internal/client/output/s3.go deleted file mode 100644 index f0527fb..0000000 --- a/internal/client/output/s3.go +++ /dev/null @@ -1,170 +0,0 @@ -package output - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log/slog" - "strings" - - "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/s3" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" -) - -var _ OutputClient = (*S3OutputClient)(nil) - -type S3OutputClient struct { - prefix string - bucketName string - s3cl *s3.Client -} - -func NewS3OutputClient(cfg *config.OutputConfig) (OutputClient, error) { - if cfg.Storage.Type != "s3" { - return nil, fmt.Errorf("invalid storage type for S3OutputClient") - } - s3cfg := cfg.Storage.Config.(*config.S3Config) - - s3cl, err := newS3Client(s3cfg) - if err != nil { - return nil, fmt.Errorf("create S3 client: %w", err) - } - - return &S3OutputClient{ - s3cl: s3cl, - bucketName: s3cfg.BucketName, - prefix: s3cfg.Prefix, - }, nil -} - -func (c *S3OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) { - key := c.prefix + path - - return &s3WriteCloser{ - key: key, - bucketName: c.bucketName, - s3cl: c.s3cl, - contentType: outputContentType, - hashOriginal: inputMetadata.Hash, - buf: &bytes.Buffer{}, - }, nil -} - -func (c *S3OutputClient) ReadMetadata(path string) (*MetadataStruct, error) { - key := c.prefix + path - - head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ - Bucket: aws.String(c.bucketName), - Key: aws.String(key), - }) - if err != nil { - return nil, fmt.Errorf("head S3 object %s: %w", key, err) - } - - metadata := MetadataStruct{ - Name: key, - StorageType: "s3", - Hash: strings.Trim(aws.ToString(head.ETag), "\""), - HashOriginal: head.Metadata["sha1-original"], - ContentType: aws.ToString(head.ContentType), - Misc: head.Metadata, - } - - if head.ContentLength != nil { - metadata.Size = *head.ContentLength - } - if head.LastModified != nil { - metadata.LastModified = *head.LastModified - metadata.FirstCreated = *head.LastModified - } - - return &metadata, nil -} - -func (c *S3OutputClient) IsMissing(path string) bool { - key := c.prefix + path - - head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{ - Bucket: aws.String(c.bucketName), - Key: aws.String(key), - }) - if err != nil { - return true - } - - headJson, _ := json.Marshal(head) - slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(headJson))) - - if head.ContentLength == nil || *head.ContentLength == 0 { - return true - } - - return false -} - -// s3WriteCloser buffers writes and uploads to S3 on Close. -type s3WriteCloser struct { - key string - bucketName string - s3cl *s3.Client - contentType string - hashOriginal string - buf *bytes.Buffer -} - -func (w *s3WriteCloser) Write(p []byte) (int, error) { - return w.buf.Write(p) -} - -func (w *s3WriteCloser) Close() error { - _, err := w.s3cl.PutObject(context.Background(), &s3.PutObjectInput{ - Bucket: aws.String(w.bucketName), - Key: aws.String(w.key), - Body: bytes.NewReader(w.buf.Bytes()), - ContentType: aws.String(w.contentType), - Metadata: map[string]string{ - "sha1-original": w.hashOriginal, - }, - }) - if err != nil { - return fmt.Errorf("put S3 object %s: %w", w.key, err) - } - return nil -} - -func newS3Client(cfg *config.S3Config) (*s3.Client, error) { - opts := []func(*awsconfig.LoadOptions) error{ - awsconfig.WithRegion(cfg.Region), - } - - if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" { - opts = append(opts, awsconfig.WithCredentialsProvider( - credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""), - )) - } - - awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) - if err != nil { - return nil, fmt.Errorf("load AWS config: %w", err) - } - - var s3Opts []func(*s3.Options) - if cfg.Endpoint != "" { - s3Opts = append(s3Opts, func(o *s3.Options) { - o.BaseEndpoint = aws.String(cfg.Endpoint) - }) - } - s3Opts = append(s3Opts, func(o *s3.Options) { - o.UsePathStyle = cfg.UsePathStyle - o.DisableLogOutputChecksumValidationSkipped = true - }) - - return s3.NewFromConfig(awsCfg, s3Opts...), nil -} diff --git a/internal/converter/converter_interface.go b/internal/converter/converter_interface.go deleted file mode 100644 index abff99d..0000000 --- a/internal/converter/converter_interface.go +++ /dev/null @@ -1,21 +0,0 @@ -package converter - -import ( - "io" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" -) - -type Converter interface { - Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error - DeductOutputPath(inputPath string) string - ReadMetadata(path string) (*output.MetadataStruct, error) - IsMissing(path string) bool -} - -var NewConverterMap = map[string]func(cfg *config.ConverterConfig) (Converter, error){ - "webp": NewWebpConverter, - "jpeg": NewJpegConverter, -} diff --git a/internal/converter/jpeg.go b/internal/converter/jpeg.go deleted file mode 100644 index a35a47c..0000000 --- a/internal/converter/jpeg.go +++ /dev/null @@ -1,114 +0,0 @@ -package converter - -import ( - "fmt" - "image" - "image/jpeg" - "image/png" - "io" - "log/slog" - "path/filepath" - "strings" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" - "github.com/kolesa-team/go-webp/webp" - "golang.org/x/image/draw" -) - -var _ Converter = (*JpegConverter)(nil) - -type JpegConverter struct { - maxWidth int - maxHeight int - extensionName string - quality int - outputClient output.OutputClient -} - -func NewJpegConverter(cfg *config.ConverterConfig) (Converter, error) { - if cfg.Type != "jpeg" { - return nil, fmt.Errorf("invalid storage type for JpegConverter") - } - jpegCfg := cfg.Config.(*config.JpegConfig) - - outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output) - if err != nil { - return nil, fmt.Errorf("fail to initialize output client: %w", err) - } - - extensionName := ".jpg" - if jpegCfg.ExtensionName != "" { - extensionName = "." + jpegCfg.ExtensionName - } - - return &JpegConverter{jpegCfg.Size.MaxWidth, jpegCfg.Size.MaxHeight, extensionName, jpegCfg.Quality, outputClient}, nil -} - -func (p *JpegConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error { - var src image.Image - - writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/jpeg") - if err != nil { - return fmt.Errorf("fail to initialize writer for output: %w", err) - } - defer writer.Close() - - switch inputMetadata.ContentType { - case "image/jpeg": - src, err = jpeg.Decode(reader) - if err != nil { - return fmt.Errorf("decode jpeg: %w", err) - } - case "image/png": - src, err = png.Decode(reader) - if err != nil { - return fmt.Errorf("decode png: %w", err) - } - case "image/webp": - src, err = webp.Decode(reader, nil) - if err != nil { - return fmt.Errorf("decode webp: %w", err) - } - default: - return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType) - } - - xCoef := 1.0 - if p.maxWidth > 0 { - xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X) - } - yCoef := 1.0 - if p.maxHeight > 0 { - yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y) - } - slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef)) - - minCoef := xCoef - if yCoef < minCoef { - minCoef = yCoef - } - - if minCoef >= 1.0 { - return jpeg.Encode(writer, src, &jpeg.Options{Quality: p.quality}) - } - - dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5))) - draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil) - - return jpeg.Encode(writer, dst, &jpeg.Options{Quality: p.quality}) -} - -func (p *JpegConverter) DeductOutputPath(inputPath string) string { - withoutExt, _ := strings.CutSuffix(inputPath, filepath.Ext(inputPath)) - return withoutExt + p.extensionName -} - -func (p *JpegConverter) ReadMetadata(path string) (*output.MetadataStruct, error) { - return p.outputClient.ReadMetadata(path) -} - -func (p *JpegConverter) IsMissing(path string) bool { - return p.outputClient.IsMissing(path) -} diff --git a/internal/converter/webp.go b/internal/converter/webp.go deleted file mode 100644 index fbffd01..0000000 --- a/internal/converter/webp.go +++ /dev/null @@ -1,118 +0,0 @@ -package converter - -import ( - "fmt" - "image" - "image/jpeg" - "image/png" - "io" - "log/slog" - "strings" - - "golang.org/x/image/draw" - - "github.com/SayaAndy/saya-today-thumbnail-generator/config" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input" - "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output" - "github.com/kolesa-team/go-webp/encoder" - "github.com/kolesa-team/go-webp/webp" -) - -var _ Converter = (*WebpConverter)(nil) - -type WebpConverter struct { - maxWidth int - maxHeight int - quality int - outputClient output.OutputClient -} - -func NewWebpConverter(cfg *config.ConverterConfig) (Converter, error) { - if cfg.Type != "webp" { - return nil, fmt.Errorf("invalid storage type for WebpConverter") - } - webpCfg := cfg.Config.(*config.WebpConfig) - - outputClient, err := output.NewOutputClientMap[cfg.Output.Storage.Type](&cfg.Output) - if err != nil { - return nil, fmt.Errorf("fail to initialize output client: %w", err) - } - - return &WebpConverter{webpCfg.Size.MaxWidth, webpCfg.Size.MaxHeight, webpCfg.Quality, outputClient}, nil -} - -func (p *WebpConverter) Process(inputMetadata *input.MetadataStruct, reader io.Reader, outputName string) error { - var src image.Image - - writer, err := p.outputClient.GetWriter(outputName, inputMetadata, "image/webp") - if err != nil { - return fmt.Errorf("fail to initialize writer for output: %w", err) - } - defer writer.Close() - - switch inputMetadata.ContentType { - case "image/jpeg": - src, err = jpeg.Decode(reader) - if err != nil { - return fmt.Errorf("decode jpeg: %w", err) - } - case "image/png": - src, err = png.Decode(reader) - if err != nil { - return fmt.Errorf("decode png: %w", err) - } - case "image/webp": - src, err = webp.Decode(reader, nil) - if err != nil { - return fmt.Errorf("decode webp: %w", err) - } - default: - return fmt.Errorf("unsupported content type: %s", inputMetadata.ContentType) - } - - opts, err := encoder.NewLossyEncoderOptions(encoder.PresetDefault, float32(p.quality)) - if err != nil { - return fmt.Errorf("create webp encoder options: %w", err) - } - - xCoef := 1.0 - if p.maxWidth > 0 { - xCoef = float64(p.maxWidth) / float64(src.Bounds().Max.X) - } - yCoef := 1.0 - if p.maxHeight > 0 { - yCoef = float64(p.maxHeight) / float64(src.Bounds().Max.Y) - } - slog.Debug("calculated coefficients", slog.Float64("x_coef", xCoef), slog.Float64("y_coef", yCoef)) - - minCoef := xCoef - if yCoef < minCoef { - minCoef = yCoef - } - - if minCoef >= 1.0 { - return webp.Encode(writer, src, opts) - } - - dst := image.NewRGBA(image.Rect(0, 0, int(float64(src.Bounds().Max.X)*minCoef+0.5), int(float64(src.Bounds().Max.Y)*minCoef+0.5))) - draw.CatmullRom.Scale(dst, dst.Rect, src, src.Bounds(), draw.Over, nil) - - return webp.Encode(writer, dst, opts) -} - -func (p *WebpConverter) DeductOutputPath(inputPath string) string { - pathParts := strings.Split(inputPath, ".") - if len(pathParts) < 2 { - return inputPath + ".webp" - } - pathParts[len(pathParts)-1] = "webp" - return strings.Join(pathParts, ".") -} - -func (p *WebpConverter) ReadMetadata(path string) (*output.MetadataStruct, error) { - return p.outputClient.ReadMetadata(path) -} - -func (p *WebpConverter) IsMissing(path string) bool { - return p.outputClient.IsMissing(path) -} |