summaryrefslogtreecommitdiff
path: root/internal/client
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat (limited to 'internal/client')
-rw-r--r--internal/client/input/b2.go135
-rw-r--r--internal/client/input/input_client_interface.go32
-rw-r--r--internal/client/input/local_unix.go108
-rw-r--r--internal/client/input/s3.go157
-rw-r--r--internal/client/output/b2.go113
-rw-r--r--internal/client/output/local_unix.go129
-rw-r--r--internal/client/output/output_client_interface.go33
-rw-r--r--internal/client/output/s3.go170
8 files changed, 877 insertions, 0 deletions
diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go
new file mode 100644
index 0000000..fe94333
--- /dev/null
+++ b/internal/client/input/b2.go
@@ -0,0 +1,135 @@
+package input
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "slices"
+ "strings"
+
+ "github.com/Backblaze/blazer/b2"
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+)
+
+var _ InputClient = (*B2InputClient)(nil)
+
+type B2InputClient struct {
+ prefix string
+ bucket *b2.Bucket
+ bucketName string
+ b2cl *b2.Client
+ knownExtensions []string
+}
+
+func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) {
+ if cfg.Storage.Type != "b2" {
+ return nil, fmt.Errorf("invalid storage type for B2InputClient")
+ }
+ b2cfg := cfg.Storage.Config.(*config.B2Config)
+
+ b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey)
+ if err != nil {
+ return nil, err
+ }
+
+ bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName)
+ if err != nil {
+ return nil, err
+ }
+
+ return &B2InputClient{b2cl: b2cl, bucket: bucket, bucketName: b2cfg.BucketName, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
+}
+
+func (c *B2InputClient) Scan() ([]string, error) {
+ filePaths := []string{}
+
+ iter := c.bucket.List(context.Background(), b2.ListPrefix(c.prefix))
+
+ for iter.Next() {
+ obj := iter.Object()
+ if obj == nil {
+ return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ }
+
+ attrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("get attributes for object: %w", err)
+ }
+
+ if attrs.Status != b2.Uploaded {
+ continue
+ }
+
+ name := obj.Name()
+
+ if len(c.knownExtensions) != 0 {
+ nameParts := strings.Split(name, ".")
+ if len(nameParts) < 2 {
+ continue
+ }
+ ext := strings.ToLower(nameParts[len(nameParts)-1])
+ if !slices.Contains(c.knownExtensions, ext) {
+ continue
+ }
+ }
+
+ filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
+ }
+
+ if err := iter.Err(); err != nil {
+ return nil, fmt.Errorf("iterate over B2 objects: %w", err)
+ }
+
+ return filePaths, nil
+}
+
+func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
+ obj := c.bucket.Object(c.prefix + path)
+ if obj == nil {
+ return nil, fmt.Errorf("object not found in B2 bucket")
+ }
+
+ attrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("get attributes for object: %w", err)
+ }
+
+ metadata := MetadataStruct{
+ Name: attrs.Name,
+ StorageType: "b2",
+ Hash: attrs.SHA1,
+ ContentType: attrs.ContentType,
+ FirstCreated: attrs.UploadTimestamp,
+ LastModified: attrs.LastModified,
+ Misc: attrs.Info,
+ Size: attrs.Size,
+ }
+
+ switch attrs.Status {
+ case b2.Uploaded:
+ metadata.Misc["b2-status"] = "Uploaded"
+ case b2.Folder:
+ metadata.Misc["b2-status"] = "Folder"
+ case b2.Hider:
+ metadata.Misc["b2-status"] = "Hider"
+ case b2.Started:
+ metadata.Misc["b2-status"] = "Started"
+ default:
+ metadata.Misc["b2-status"] = "Unknown"
+ }
+
+ return &metadata, nil
+}
+
+func (c *B2InputClient) ID(path string) string {
+ return fmt.Sprintf("b2://%s/%s%s", c.bucketName, c.prefix, path)
+}
+
+func (c *B2InputClient) GetReader(path string) (io.ReadCloser, error) {
+ obj := c.bucket.Object(c.prefix + path)
+ if obj == nil {
+ return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ }
+
+ return obj.NewReader(context.Background()), nil
+}
diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go
new file mode 100644
index 0000000..dca64db
--- /dev/null
+++ b/internal/client/input/input_client_interface.go
@@ -0,0 +1,32 @@
+package input
+
+import (
+ "io"
+ "time"
+
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+)
+
+type InputClient interface {
+ Scan() ([]string, error)
+ ReadMetadata(string) (*MetadataStruct, error)
+ GetReader(string) (io.ReadCloser, error)
+ ID(path string) string
+}
+
+type MetadataStruct struct {
+ Name string
+ StorageType string
+ Hash string
+ ContentType string
+ FirstCreated time.Time
+ LastModified time.Time
+ Size int64
+ Misc map[string]string
+}
+
+var NewInputClientMap = map[string]func(cfg *config.InputConfig) (InputClient, error){
+ "b2": NewB2InputClient,
+ "s3": NewS3InputClient,
+ "local-unix": NewLocalUnixInputClient,
+}
diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go
new file mode 100644
index 0000000..21b08a9
--- /dev/null
+++ b/internal/client/input/local_unix.go
@@ -0,0 +1,108 @@
+package input
+
+import (
+ "fmt"
+ "io"
+ "log/slog"
+ "mime"
+ "os"
+ "slices"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+)
+
+var _ InputClient = (*LocalUnixInputClient)(nil)
+
+type LocalUnixInputClient struct {
+ path string
+ maxDepth int
+ knownExtensions []string
+}
+
+func NewLocalUnixInputClient(cfg *config.InputConfig) (InputClient, error) {
+ if cfg.Storage.Type != "local-unix" {
+ return nil, fmt.Errorf("invalid storage type for LocalUnixInputClient")
+ }
+ localCfg := cfg.Storage.Config.(*config.InputLocalUnixConfig)
+
+ return &LocalUnixInputClient{
+ path: localCfg.Path,
+ maxDepth: localCfg.MaxDepth,
+ knownExtensions: cfg.KnownExtensions,
+ }, nil
+}
+
+func (c *LocalUnixInputClient) Scan() ([]string, error) {
+ return c.recursiveScan(c.path, c.maxDepth)
+}
+
+func (c *LocalUnixInputClient) recursiveScan(dir string, depth int) ([]string, error) {
+ filePaths := make([]string, 0)
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return nil, fmt.Errorf("fail to read directory: %w", err)
+ }
+
+ for _, entry := range entries {
+ if entry.IsDir() && depth > 0 {
+ subFilePaths, err := c.recursiveScan(dir+entry.Name()+"/", depth-1)
+ if err != nil {
+ return nil, fmt.Errorf("fail to scan subdirectory '%s': %w", entry.Name(), err)
+ }
+ filePaths = append(filePaths, subFilePaths...)
+ } else if !entry.IsDir() {
+ nameParts := strings.Split(entry.Name(), ".")
+ if len(nameParts) < 2 {
+ continue
+ }
+ if slices.Contains(c.knownExtensions, strings.ToLower(nameParts[len(nameParts)-1])) {
+ filePaths = append(filePaths, strings.TrimPrefix(dir+entry.Name(), c.path))
+ }
+ fmt.Println(entry.Name())
+ }
+ }
+
+ return filePaths, nil
+}
+
+func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error) {
+ nodePathParts := strings.Split(path, "/")
+ nodeName := nodePathParts[len(nodePathParts)-1]
+ nodeNameParts := strings.Split(nodeName, ".")
+ nodeExt := ""
+ if len(nodeNameParts) >= 2 {
+ nodeExt = nodeNameParts[len(nodeNameParts)-1]
+ }
+ slog.Debug("got a file extension", slog.String("extension", nodeExt), slog.String("path", path), slog.String("filename", nodeName))
+
+ fileInfo, err := os.Stat(c.path + path)
+ if err != nil {
+ return nil, fmt.Errorf("fail to read file info: %w", err)
+ }
+
+ stat_t := fileInfo.Sys().(*syscall.Stat_t)
+ creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
+
+ return &MetadataStruct{
+ Name: fileInfo.Name(),
+ StorageType: "local-unix",
+ Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16),
+ ContentType: mime.TypeByExtension("." + nodeExt),
+ FirstCreated: creationTime,
+ LastModified: fileInfo.ModTime(),
+ Size: fileInfo.Size(),
+ Misc: map[string]string{},
+ }, nil
+}
+
+func (c *LocalUnixInputClient) ID(path string) string {
+ return fmt.Sprintf("local-unix://%s%s", c.path, path)
+}
+
+func (c *LocalUnixInputClient) GetReader(path string) (io.ReadCloser, error) {
+ return os.Open(c.path + path)
+}
diff --git a/internal/client/input/s3.go b/internal/client/input/s3.go
new file mode 100644
index 0000000..00684c2
--- /dev/null
+++ b/internal/client/input/s3.go
@@ -0,0 +1,157 @@
+package input
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "slices"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+)
+
+var _ InputClient = (*S3InputClient)(nil)
+
+type S3InputClient struct {
+ prefix string
+ bucketName string
+ s3cl *s3.Client
+ knownExtensions []string
+}
+
+func NewS3InputClient(cfg *config.InputConfig) (InputClient, error) {
+ if cfg.Storage.Type != "s3" {
+ return nil, fmt.Errorf("invalid storage type for S3InputClient")
+ }
+ s3cfg := cfg.Storage.Config.(*config.S3Config)
+
+ s3cl, err := newS3Client(s3cfg)
+ if err != nil {
+ return nil, fmt.Errorf("create S3 client: %w", err)
+ }
+
+ return &S3InputClient{
+ s3cl: s3cl,
+ bucketName: s3cfg.BucketName,
+ prefix: s3cfg.Prefix,
+ knownExtensions: cfg.KnownExtensions,
+ }, nil
+}
+
+func (c *S3InputClient) Scan() ([]string, error) {
+ var filePaths []string
+
+ paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucketName),
+ Prefix: aws.String(c.prefix),
+ })
+
+ for paginator.HasMorePages() {
+ output, err := paginator.NextPage(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("list S3 objects: %w", err)
+ }
+
+ for _, obj := range output.Contents {
+ name := aws.ToString(obj.Key)
+
+ if len(c.knownExtensions) != 0 {
+ nameParts := strings.Split(name, ".")
+ if len(nameParts) < 2 {
+ continue
+ }
+ ext := strings.ToLower(nameParts[len(nameParts)-1])
+ if !slices.Contains(c.knownExtensions, ext) {
+ continue
+ }
+ }
+
+ filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
+ }
+ }
+
+ return filePaths, nil
+}
+
+func (c *S3InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
+ key := c.prefix + path
+
+ head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("head S3 object %s: %w", key, err)
+ }
+
+ metadata := MetadataStruct{
+ Name: key,
+ StorageType: "s3",
+ Hash: strings.Trim(aws.ToString(head.ETag), "\""),
+ ContentType: aws.ToString(head.ContentType),
+ Misc: head.Metadata,
+ }
+
+ if head.ContentLength != nil {
+ metadata.Size = *head.ContentLength
+ }
+ if head.LastModified != nil {
+ metadata.LastModified = *head.LastModified
+ metadata.FirstCreated = *head.LastModified
+ }
+
+ return &metadata, nil
+}
+
+func (c *S3InputClient) ID(path string) string {
+ return fmt.Sprintf("s3://%s/%s%s", c.bucketName, c.prefix, path)
+}
+
+func (c *S3InputClient) GetReader(path string) (io.ReadCloser, error) {
+ key := c.prefix + path
+
+ output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("get S3 object %s: %w", key, err)
+ }
+
+ return output.Body, nil
+}
+
+func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
+ opts := []func(*awsconfig.LoadOptions) error{
+ awsconfig.WithRegion(cfg.Region),
+ }
+
+ if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
+ opts = append(opts, awsconfig.WithCredentialsProvider(
+ credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
+ ))
+ }
+
+ awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
+ if err != nil {
+ return nil, fmt.Errorf("load AWS config: %w", err)
+ }
+
+ var s3Opts []func(*s3.Options)
+ if cfg.Endpoint != "" {
+ s3Opts = append(s3Opts, func(o *s3.Options) {
+ o.BaseEndpoint = aws.String(cfg.Endpoint)
+ })
+ }
+ s3Opts = append(s3Opts, func(o *s3.Options) {
+ o.UsePathStyle = cfg.UsePathStyle
+ o.DisableLogOutputChecksumValidationSkipped = true
+ })
+
+ return s3.NewFromConfig(awsCfg, s3Opts...), nil
+}
diff --git a/internal/client/output/b2.go b/internal/client/output/b2.go
new file mode 100644
index 0000000..0616368
--- /dev/null
+++ b/internal/client/output/b2.go
@@ -0,0 +1,113 @@
+package output
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+
+ "github.com/Backblaze/blazer/b2"
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+ "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
+)
+
+var _ OutputClient = (*B2OutputClient)(nil)
+
+type B2OutputClient struct {
+ prefix string
+ bucket *b2.Bucket
+ b2cl *b2.Client
+}
+
+func NewB2OutputClient(cfg *config.OutputConfig) (OutputClient, error) {
+ if cfg.Storage.Type != "b2" {
+ return nil, fmt.Errorf("invalid storage type for B2OutputClient")
+ }
+ b2cfg := cfg.Storage.Config.(*config.B2Config)
+
+ b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey)
+ if err != nil {
+ return nil, err
+ }
+
+ bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName)
+ if err != nil {
+ return nil, err
+ }
+
+ return &B2OutputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
+}
+
+func (c *B2OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) {
+ obj := c.bucket.Object(c.prefix + path)
+ if obj == nil {
+ return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ }
+
+ attrs := &b2.Attrs{Info: make(map[string]string)}
+ attrs.Info["sha1-original"] = inputMetadata.Hash
+ attrs.ContentType = outputContentType
+
+ return obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs)), nil
+}
+
+func (c *B2OutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
+ obj := c.bucket.Object(c.prefix + path)
+ if obj == nil {
+ return nil, fmt.Errorf("failed to reference object in B2 bucket")
+ }
+
+ attrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("get attributes for object: %w", err)
+ }
+
+ metadata := MetadataStruct{
+ Name: attrs.Name,
+ StorageType: "b2",
+ Hash: attrs.SHA1,
+ HashOriginal: attrs.Info["sha1-original"],
+ ContentType: attrs.ContentType,
+ FirstCreated: attrs.UploadTimestamp,
+ LastModified: attrs.LastModified,
+ Misc: attrs.Info,
+ Size: attrs.Size,
+ }
+
+ switch attrs.Status {
+ case b2.Uploaded:
+ metadata.Misc["b2-status"] = "Uploaded"
+ case b2.Folder:
+ metadata.Misc["b2-status"] = "Folder"
+ case b2.Hider:
+ metadata.Misc["b2-status"] = "Hider"
+ case b2.Started:
+ metadata.Misc["b2-status"] = "Started"
+ default:
+ metadata.Misc["b2-status"] = "Unknown"
+ }
+
+ return &metadata, nil
+}
+
+func (c *B2OutputClient) IsMissing(path string) bool {
+ obj := c.bucket.Object(c.prefix + path)
+ if obj == nil {
+ return true
+ }
+
+ attrs, err := obj.Attrs(context.Background())
+ if err != nil {
+ return true
+ }
+
+ attrsJson, _ := json.Marshal(attrs)
+ slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(attrsJson)))
+
+ if attrs.Size == 0 {
+ return true
+ }
+
+ return attrs.Status == b2.Hider
+}
diff --git a/internal/client/output/local_unix.go b/internal/client/output/local_unix.go
new file mode 100644
index 0000000..6f9a7b3
--- /dev/null
+++ b/internal/client/output/local_unix.go
@@ -0,0 +1,129 @@
+package output
+
+import (
+ "fmt"
+ "io"
+ "log/slog"
+ "mime"
+ "os"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+ "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
+ "golang.org/x/sys/unix"
+)
+
+var _ OutputClient = (*LocalUnixOutputClient)(nil)
+
+type LocalUnixOutputClient struct {
+ path string
+ fileMode uint32
+ dirMode uint32
+ attrMode string
+}
+
+func NewLocalUnixOutputClient(cfg *config.OutputConfig) (OutputClient, error) {
+ if cfg.Storage.Type != "local-unix" {
+ return nil, fmt.Errorf("invalid storage type for LocalUnixOutputClient")
+ }
+ localCfg := cfg.Storage.Config.(*config.OutputLocalUnixConfig)
+
+ fpm, err := strconv.ParseInt(localCfg.FilePermissionMode, 8, 32)
+ if err != nil {
+ return nil, fmt.Errorf("fail to parse file permission mode as an octal number: %w", err)
+ }
+
+ dpm, err := strconv.ParseInt(localCfg.DirPermissionMode, 8, 32)
+ if err != nil {
+ return nil, fmt.Errorf("fail to parse directory permission mode as an octal number: %w", err)
+ }
+
+ return &LocalUnixOutputClient{localCfg.Path, uint32(fpm), uint32(dpm), localCfg.AttributesImplementation}, nil
+}
+
+func (c *LocalUnixOutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, _ string) (io.WriteCloser, error) {
+ pathSegments := strings.Split(path, "/")
+ dirpath := strings.Join(pathSegments[0:len(pathSegments)-1], "/")
+ if err := os.MkdirAll(c.path+dirpath, os.FileMode(c.dirMode)); err != nil {
+ return nil, fmt.Errorf("fail to mkdir parent directories for a path: %w", err)
+ }
+
+ var err error
+ if _, err = os.Stat(c.path + path); err != nil && !os.IsNotExist(err) {
+ return nil, fmt.Errorf("fail to stat a file, although file exists: %w", err)
+ }
+ if os.IsNotExist(err) {
+ f, err := os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE, os.FileMode(c.fileMode))
+ if err != nil {
+ return nil, fmt.Errorf("fail to create a file: %w", err)
+ }
+ f.Close()
+ }
+
+ switch c.attrMode {
+ case "xattr":
+ if err := unix.Setxattr(c.path+path, "user.originalfile.mddate", []byte(strconv.FormatInt(inputMetadata.LastModified.Unix(), 16)), 0); err != nil {
+ return nil, fmt.Errorf("fail to write user.originalfile.mddate xattribute: %w", err)
+ }
+ case "none":
+ default:
+ return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode)
+ }
+
+ return os.OpenFile(c.path+path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(c.fileMode))
+}
+
+func (c *LocalUnixOutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
+ nodePathParts := strings.Split(path, "/")
+ nodeName := nodePathParts[len(nodePathParts)-1]
+ nodeNameParts := strings.Split(nodeName, ".")
+ nodeExt := ""
+ if len(nodeNameParts) >= 2 {
+ nodeExt = nodeNameParts[len(nodeNameParts)-1]
+ }
+
+ fileInfo, err := os.Stat(c.path + path)
+ if err != nil {
+ return nil, fmt.Errorf("fail to read file info: %w", err)
+ }
+
+ stat_t := fileInfo.Sys().(*syscall.Stat_t)
+ creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
+
+ mddateOriginal := make([]byte, 0)
+ switch c.attrMode {
+ case "xattr":
+ sz, err := unix.Getxattr(c.path+path, "user.originalfile.mddate", nil)
+ if err != nil {
+ slog.Warn("fail to get size of user.originalfile.mddate attribute, proceeding as if no such attribute is there", slog.String("error", err.Error()))
+ break
+ }
+ mddateOriginal = make([]byte, sz)
+ if _, err = unix.Getxattr(c.path+path, "user.originalfile.mddate", mddateOriginal); err != nil {
+ return nil, fmt.Errorf("fail to get user.originalfile.mddate attribute: %w", err)
+ }
+ case "none":
+ default:
+ return nil, fmt.Errorf("unknown attributes implementation: %s", c.attrMode)
+ }
+
+ return &MetadataStruct{
+ Name: fileInfo.Name(),
+ StorageType: "local-unix",
+ Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16),
+ HashOriginal: string(mddateOriginal),
+ ContentType: mime.TypeByExtension("." + nodeExt),
+ FirstCreated: creationTime,
+ LastModified: fileInfo.ModTime(),
+ Size: fileInfo.Size(),
+ Misc: map[string]string{},
+ }, nil
+}
+
+func (c *LocalUnixOutputClient) IsMissing(path string) bool {
+ _, err := os.Stat(c.path + path)
+ return err != nil
+}
diff --git a/internal/client/output/output_client_interface.go b/internal/client/output/output_client_interface.go
new file mode 100644
index 0000000..4415aa4
--- /dev/null
+++ b/internal/client/output/output_client_interface.go
@@ -0,0 +1,33 @@
+package output
+
+import (
+ "io"
+ "time"
+
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+ "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
+)
+
+type OutputClient interface {
+ GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error)
+ ReadMetadata(path string) (*MetadataStruct, error)
+ IsMissing(path string) bool
+}
+
+type MetadataStruct struct {
+ Name string
+ StorageType string
+ Hash string
+ HashOriginal string
+ ContentType string
+ FirstCreated time.Time
+ LastModified time.Time
+ Size int64
+ Misc map[string]string
+}
+
+var NewOutputClientMap = map[string]func(cfg *config.OutputConfig) (OutputClient, error){
+ "b2": NewB2OutputClient,
+ "s3": NewS3OutputClient,
+ "local-unix": NewLocalUnixOutputClient,
+}
diff --git a/internal/client/output/s3.go b/internal/client/output/s3.go
new file mode 100644
index 0000000..f0527fb
--- /dev/null
+++ b/internal/client/output/s3.go
@@ -0,0 +1,170 @@
+package output
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+ "github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
+)
+
+var _ OutputClient = (*S3OutputClient)(nil)
+
+type S3OutputClient struct {
+ prefix string
+ bucketName string
+ s3cl *s3.Client
+}
+
+func NewS3OutputClient(cfg *config.OutputConfig) (OutputClient, error) {
+ if cfg.Storage.Type != "s3" {
+ return nil, fmt.Errorf("invalid storage type for S3OutputClient")
+ }
+ s3cfg := cfg.Storage.Config.(*config.S3Config)
+
+ s3cl, err := newS3Client(s3cfg)
+ if err != nil {
+ return nil, fmt.Errorf("create S3 client: %w", err)
+ }
+
+ return &S3OutputClient{
+ s3cl: s3cl,
+ bucketName: s3cfg.BucketName,
+ prefix: s3cfg.Prefix,
+ }, nil
+}
+
+func (c *S3OutputClient) GetWriter(path string, inputMetadata *input.MetadataStruct, outputContentType string) (io.WriteCloser, error) {
+ key := c.prefix + path
+
+ return &s3WriteCloser{
+ key: key,
+ bucketName: c.bucketName,
+ s3cl: c.s3cl,
+ contentType: outputContentType,
+ hashOriginal: inputMetadata.Hash,
+ buf: &bytes.Buffer{},
+ }, nil
+}
+
+func (c *S3OutputClient) ReadMetadata(path string) (*MetadataStruct, error) {
+ key := c.prefix + path
+
+ head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("head S3 object %s: %w", key, err)
+ }
+
+ metadata := MetadataStruct{
+ Name: key,
+ StorageType: "s3",
+ Hash: strings.Trim(aws.ToString(head.ETag), "\""),
+ HashOriginal: head.Metadata["sha1-original"],
+ ContentType: aws.ToString(head.ContentType),
+ Misc: head.Metadata,
+ }
+
+ if head.ContentLength != nil {
+ metadata.Size = *head.ContentLength
+ }
+ if head.LastModified != nil {
+ metadata.LastModified = *head.LastModified
+ metadata.FirstCreated = *head.LastModified
+ }
+
+ return &metadata, nil
+}
+
+func (c *S3OutputClient) IsMissing(path string) bool {
+ key := c.prefix + path
+
+ head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return true
+ }
+
+ headJson, _ := json.Marshal(head)
+ slog.Debug("got object attrs", slog.String("path", path), slog.String("attrs", string(headJson)))
+
+ if head.ContentLength == nil || *head.ContentLength == 0 {
+ return true
+ }
+
+ return false
+}
+
+// s3WriteCloser buffers writes and uploads to S3 on Close.
+type s3WriteCloser struct {
+ key string
+ bucketName string
+ s3cl *s3.Client
+ contentType string
+ hashOriginal string
+ buf *bytes.Buffer
+}
+
+func (w *s3WriteCloser) Write(p []byte) (int, error) {
+ return w.buf.Write(p)
+}
+
+func (w *s3WriteCloser) Close() error {
+ _, err := w.s3cl.PutObject(context.Background(), &s3.PutObjectInput{
+ Bucket: aws.String(w.bucketName),
+ Key: aws.String(w.key),
+ Body: bytes.NewReader(w.buf.Bytes()),
+ ContentType: aws.String(w.contentType),
+ Metadata: map[string]string{
+ "sha1-original": w.hashOriginal,
+ },
+ })
+ if err != nil {
+ return fmt.Errorf("put S3 object %s: %w", w.key, err)
+ }
+ return nil
+}
+
+func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
+ opts := []func(*awsconfig.LoadOptions) error{
+ awsconfig.WithRegion(cfg.Region),
+ }
+
+ if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
+ opts = append(opts, awsconfig.WithCredentialsProvider(
+ credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
+ ))
+ }
+
+ awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
+ if err != nil {
+ return nil, fmt.Errorf("load AWS config: %w", err)
+ }
+
+ var s3Opts []func(*s3.Options)
+ if cfg.Endpoint != "" {
+ s3Opts = append(s3Opts, func(o *s3.Options) {
+ o.BaseEndpoint = aws.String(cfg.Endpoint)
+ })
+ }
+ s3Opts = append(s3Opts, func(o *s3.Options) {
+ o.UsePathStyle = cfg.UsePathStyle
+ o.DisableLogOutputChecksumValidationSkipped = true
+ })
+
+ return s3.NewFromConfig(awsCfg, s3Opts...), nil
+}
generated by cgitext v1.3.1+16 (git 2.54.0) at 2026-09-07 19:41:13 +0000