summaryrefslogtreecommitdiff
path: root/internal/client/input
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat (limited to 'internal/client/input')
-rw-r--r--internal/client/input/b2.go7
-rw-r--r--internal/client/input/input_client_interface.go2
-rw-r--r--internal/client/input/local_unix.go6
-rw-r--r--internal/client/input/s3.go157
4 files changed, 170 insertions, 2 deletions
diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go
index db68268..fe94333 100644
--- a/internal/client/input/b2.go
+++ b/internal/client/input/b2.go
@@ -16,6 +16,7 @@ var _ InputClient = (*B2InputClient)(nil)
type B2InputClient struct {
prefix string
bucket *b2.Bucket
+ bucketName string
b2cl *b2.Client
knownExtensions []string
}
@@ -36,7 +37,7 @@ func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) {
return nil, err
}
- return &B2InputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
+ return &B2InputClient{b2cl: b2cl, bucket: bucket, bucketName: b2cfg.BucketName, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
}
func (c *B2InputClient) Scan() ([]string, error) {
@@ -120,6 +121,10 @@ func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
return &metadata, nil
}
+func (c *B2InputClient) ID(path string) string {
+ return fmt.Sprintf("b2://%s/%s%s", c.bucketName, c.prefix, path)
+}
+
func (c *B2InputClient) GetReader(path string) (io.ReadCloser, error) {
obj := c.bucket.Object(c.prefix + path)
if obj == nil {
diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go
index 4930aa0..dca64db 100644
--- a/internal/client/input/input_client_interface.go
+++ b/internal/client/input/input_client_interface.go
@@ -11,6 +11,7 @@ type InputClient interface {
Scan() ([]string, error)
ReadMetadata(string) (*MetadataStruct, error)
GetReader(string) (io.ReadCloser, error)
+ ID(path string) string
}
type MetadataStruct struct {
@@ -26,5 +27,6 @@ type MetadataStruct struct {
var NewInputClientMap = map[string]func(cfg *config.InputConfig) (InputClient, error){
"b2": NewB2InputClient,
+ "s3": NewS3InputClient,
"local-unix": NewLocalUnixInputClient,
}
diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go
index 27a7947..21b08a9 100644
--- a/internal/client/input/local_unix.go
+++ b/internal/client/input/local_unix.go
@@ -85,7 +85,7 @@ func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error
}
stat_t := fileInfo.Sys().(*syscall.Stat_t)
- creationTime := time.Unix(stat_t.Ctim.Sec, stat_t.Ctim.Nsec)
+ creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
return &MetadataStruct{
Name: fileInfo.Name(),
@@ -99,6 +99,10 @@ func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error
}, nil
}
+func (c *LocalUnixInputClient) ID(path string) string {
+ return fmt.Sprintf("local-unix://%s%s", c.path, path)
+}
+
func (c *LocalUnixInputClient) GetReader(path string) (io.ReadCloser, error) {
return os.Open(c.path + path)
}
diff --git a/internal/client/input/s3.go b/internal/client/input/s3.go
new file mode 100644
index 0000000..00684c2
--- /dev/null
+++ b/internal/client/input/s3.go
@@ -0,0 +1,157 @@
+package input
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "slices"
+ "strings"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsconfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+
+ "github.com/SayaAndy/saya-today-thumbnail-generator/config"
+)
+
+var _ InputClient = (*S3InputClient)(nil)
+
+type S3InputClient struct {
+ prefix string
+ bucketName string
+ s3cl *s3.Client
+ knownExtensions []string
+}
+
+func NewS3InputClient(cfg *config.InputConfig) (InputClient, error) {
+ if cfg.Storage.Type != "s3" {
+ return nil, fmt.Errorf("invalid storage type for S3InputClient")
+ }
+ s3cfg := cfg.Storage.Config.(*config.S3Config)
+
+ s3cl, err := newS3Client(s3cfg)
+ if err != nil {
+ return nil, fmt.Errorf("create S3 client: %w", err)
+ }
+
+ return &S3InputClient{
+ s3cl: s3cl,
+ bucketName: s3cfg.BucketName,
+ prefix: s3cfg.Prefix,
+ knownExtensions: cfg.KnownExtensions,
+ }, nil
+}
+
+func (c *S3InputClient) Scan() ([]string, error) {
+ var filePaths []string
+
+ paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
+ Bucket: aws.String(c.bucketName),
+ Prefix: aws.String(c.prefix),
+ })
+
+ for paginator.HasMorePages() {
+ output, err := paginator.NextPage(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("list S3 objects: %w", err)
+ }
+
+ for _, obj := range output.Contents {
+ name := aws.ToString(obj.Key)
+
+ if len(c.knownExtensions) != 0 {
+ nameParts := strings.Split(name, ".")
+ if len(nameParts) < 2 {
+ continue
+ }
+ ext := strings.ToLower(nameParts[len(nameParts)-1])
+ if !slices.Contains(c.knownExtensions, ext) {
+ continue
+ }
+ }
+
+ filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
+ }
+ }
+
+ return filePaths, nil
+}
+
+func (c *S3InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
+ key := c.prefix + path
+
+ head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("head S3 object %s: %w", key, err)
+ }
+
+ metadata := MetadataStruct{
+ Name: key,
+ StorageType: "s3",
+ Hash: strings.Trim(aws.ToString(head.ETag), "\""),
+ ContentType: aws.ToString(head.ContentType),
+ Misc: head.Metadata,
+ }
+
+ if head.ContentLength != nil {
+ metadata.Size = *head.ContentLength
+ }
+ if head.LastModified != nil {
+ metadata.LastModified = *head.LastModified
+ metadata.FirstCreated = *head.LastModified
+ }
+
+ return &metadata, nil
+}
+
+func (c *S3InputClient) ID(path string) string {
+ return fmt.Sprintf("s3://%s/%s%s", c.bucketName, c.prefix, path)
+}
+
+func (c *S3InputClient) GetReader(path string) (io.ReadCloser, error) {
+ key := c.prefix + path
+
+ output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
+ Bucket: aws.String(c.bucketName),
+ Key: aws.String(key),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("get S3 object %s: %w", key, err)
+ }
+
+ return output.Body, nil
+}
+
+func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
+ opts := []func(*awsconfig.LoadOptions) error{
+ awsconfig.WithRegion(cfg.Region),
+ }
+
+ if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
+ opts = append(opts, awsconfig.WithCredentialsProvider(
+ credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
+ ))
+ }
+
+ awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
+ if err != nil {
+ return nil, fmt.Errorf("load AWS config: %w", err)
+ }
+
+ var s3Opts []func(*s3.Options)
+ if cfg.Endpoint != "" {
+ s3Opts = append(s3Opts, func(o *s3.Options) {
+ o.BaseEndpoint = aws.String(cfg.Endpoint)
+ })
+ }
+ s3Opts = append(s3Opts, func(o *s3.Options) {
+ o.UsePathStyle = cfg.UsePathStyle
+ o.DisableLogOutputChecksumValidationSkipped = true
+ })
+
+ return s3.NewFromConfig(awsCfg, s3Opts...), nil
+}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-20 10:17:28 +0000