summaryrefslogtreecommitdiff
path: root/internal/client/input
diff refs
from: back
to: back
| flip
diff options
context:
space:
mode:
Diffstat (limited to 'internal/client/input')
-rw-r--r--internal/client/input/b2.go52
-rw-r--r--internal/client/input/input_client_interface.go25
-rw-r--r--internal/client/input/local_unix.go108
-rw-r--r--internal/client/input/s3.go157
4 files changed, 25 insertions, 317 deletions
diff --git a/internal/client/input/b2.go b/internal/client/input/b2.go
index fe94333..67fb14e 100644
--- a/internal/client/input/b2.go
+++ b/internal/client/input/b2.go
@@ -16,12 +16,11 @@ var _ InputClient = (*B2InputClient)(nil)
type B2InputClient struct {
prefix string
bucket *b2.Bucket
- bucketName string
b2cl *b2.Client
knownExtensions []string
}
-func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) {
+func NewB2InputClient(cfg *config.InputConfig) (*B2InputClient, error) {
if cfg.Storage.Type != "b2" {
return nil, fmt.Errorf("invalid storage type for B2InputClient")
}
@@ -37,7 +36,7 @@ func NewB2InputClient(cfg *config.InputConfig) (InputClient, error) {
return nil, err
}
- return &B2InputClient{b2cl: b2cl, bucket: bucket, bucketName: b2cfg.BucketName, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
+ return &B2InputClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, knownExtensions: cfg.KnownExtensions}, nil
}
func (c *B2InputClient) Scan() ([]string, error) {
@@ -73,7 +72,7 @@ func (c *B2InputClient) Scan() ([]string, error) {
}
}
- filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
+ filePaths = append(filePaths, name)
}
if err := iter.Err(); err != nil {
@@ -83,8 +82,8 @@ func (c *B2InputClient) Scan() ([]string, error) {
return filePaths, nil
}
-func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- obj := c.bucket.Object(c.prefix + path)
+func (c *B2InputClient) ReadMetadata(path string) (map[string]string, error) {
+ obj := c.bucket.Object(path)
if obj == nil {
return nil, fmt.Errorf("object not found in B2 bucket")
}
@@ -94,42 +93,37 @@ func (c *B2InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
return nil, fmt.Errorf("get attributes for object: %w", err)
}
- metadata := MetadataStruct{
- Name: attrs.Name,
- StorageType: "b2",
- Hash: attrs.SHA1,
- ContentType: attrs.ContentType,
- FirstCreated: attrs.UploadTimestamp,
- LastModified: attrs.LastModified,
- Misc: attrs.Info,
- Size: attrs.Size,
- }
+ metadata := attrs.Info
+ metadata["Name"] = attrs.Name
+ metadata["Size"] = fmt.Sprintf("%d", attrs.Size)
+ metadata["ContentType"] = attrs.ContentType
+ metadata["LastModified"] = attrs.LastModified.Format("2006-01-02T15:04:05Z")
+ metadata["SHA1"] = attrs.SHA1
+ metadata["UploadTimestamp"] = attrs.UploadTimestamp.Format("2006-01-02T15:04:05Z")
switch attrs.Status {
case b2.Uploaded:
- metadata.Misc["b2-status"] = "Uploaded"
+ metadata["Status"] = "Uploaded"
case b2.Folder:
- metadata.Misc["b2-status"] = "Folder"
+ metadata["Status"] = "Folder"
case b2.Hider:
- metadata.Misc["b2-status"] = "Hider"
+ metadata["Status"] = "Hider"
case b2.Started:
- metadata.Misc["b2-status"] = "Started"
+ metadata["Status"] = "Started"
default:
- metadata.Misc["b2-status"] = "Unknown"
+ metadata["Status"] = "Unknown"
}
- return &metadata, nil
-}
-
-func (c *B2InputClient) ID(path string) string {
- return fmt.Sprintf("b2://%s/%s%s", c.bucketName, c.prefix, path)
+ return metadata, nil
}
-func (c *B2InputClient) GetReader(path string) (io.ReadCloser, error) {
- obj := c.bucket.Object(c.prefix + path)
+func (c *B2InputClient) GetReader(path string) (io.Reader, error) {
+ obj := c.bucket.Object(path)
if obj == nil {
return nil, fmt.Errorf("failed to reference object in B2 bucket")
}
- return obj.NewReader(context.Background()), nil
+ reader := obj.NewReader(context.Background())
+
+ return reader, nil
}
diff --git a/internal/client/input/input_client_interface.go b/internal/client/input/input_client_interface.go
index dca64db..4bbbae5 100644
--- a/internal/client/input/input_client_interface.go
+++ b/internal/client/input/input_client_interface.go
@@ -2,31 +2,10 @@ package input
import (
"io"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
)
type InputClient interface {
Scan() ([]string, error)
- ReadMetadata(string) (*MetadataStruct, error)
- GetReader(string) (io.ReadCloser, error)
- ID(path string) string
-}
-
-type MetadataStruct struct {
- Name string
- StorageType string
- Hash string
- ContentType string
- FirstCreated time.Time
- LastModified time.Time
- Size int64
- Misc map[string]string
-}
-
-var NewInputClientMap = map[string]func(cfg *config.InputConfig) (InputClient, error){
- "b2": NewB2InputClient,
- "s3": NewS3InputClient,
- "local-unix": NewLocalUnixInputClient,
+ ReadMetadata(string) (map[string]string, error)
+ GetReader(string) (io.Reader, error)
}
diff --git a/internal/client/input/local_unix.go b/internal/client/input/local_unix.go
deleted file mode 100644
index 21b08a9..0000000
--- a/internal/client/input/local_unix.go
+++ /dev/null
@@ -1,108 +0,0 @@
-package input
-
-import (
- "fmt"
- "io"
- "log/slog"
- "mime"
- "os"
- "slices"
- "strconv"
- "strings"
- "syscall"
- "time"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-var _ InputClient = (*LocalUnixInputClient)(nil)
-
-type LocalUnixInputClient struct {
- path string
- maxDepth int
- knownExtensions []string
-}
-
-func NewLocalUnixInputClient(cfg *config.InputConfig) (InputClient, error) {
- if cfg.Storage.Type != "local-unix" {
- return nil, fmt.Errorf("invalid storage type for LocalUnixInputClient")
- }
- localCfg := cfg.Storage.Config.(*config.InputLocalUnixConfig)
-
- return &LocalUnixInputClient{
- path: localCfg.Path,
- maxDepth: localCfg.MaxDepth,
- knownExtensions: cfg.KnownExtensions,
- }, nil
-}
-
-func (c *LocalUnixInputClient) Scan() ([]string, error) {
- return c.recursiveScan(c.path, c.maxDepth)
-}
-
-func (c *LocalUnixInputClient) recursiveScan(dir string, depth int) ([]string, error) {
- filePaths := make([]string, 0)
- entries, err := os.ReadDir(dir)
- if err != nil {
- return nil, fmt.Errorf("fail to read directory: %w", err)
- }
-
- for _, entry := range entries {
- if entry.IsDir() && depth > 0 {
- subFilePaths, err := c.recursiveScan(dir+entry.Name()+"/", depth-1)
- if err != nil {
- return nil, fmt.Errorf("fail to scan subdirectory '%s': %w", entry.Name(), err)
- }
- filePaths = append(filePaths, subFilePaths...)
- } else if !entry.IsDir() {
- nameParts := strings.Split(entry.Name(), ".")
- if len(nameParts) < 2 {
- continue
- }
- if slices.Contains(c.knownExtensions, strings.ToLower(nameParts[len(nameParts)-1])) {
- filePaths = append(filePaths, strings.TrimPrefix(dir+entry.Name(), c.path))
- }
- fmt.Println(entry.Name())
- }
- }
-
- return filePaths, nil
-}
-
-func (c *LocalUnixInputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- nodePathParts := strings.Split(path, "/")
- nodeName := nodePathParts[len(nodePathParts)-1]
- nodeNameParts := strings.Split(nodeName, ".")
- nodeExt := ""
- if len(nodeNameParts) >= 2 {
- nodeExt = nodeNameParts[len(nodeNameParts)-1]
- }
- slog.Debug("got a file extension", slog.String("extension", nodeExt), slog.String("path", path), slog.String("filename", nodeName))
-
- fileInfo, err := os.Stat(c.path + path)
- if err != nil {
- return nil, fmt.Errorf("fail to read file info: %w", err)
- }
-
- stat_t := fileInfo.Sys().(*syscall.Stat_t)
- creationTime := time.Unix(stat_t.Ctimespec.Sec, stat_t.Ctimespec.Nsec)
-
- return &MetadataStruct{
- Name: fileInfo.Name(),
- StorageType: "local-unix",
- Hash: strconv.FormatInt(fileInfo.ModTime().Unix(), 16),
- ContentType: mime.TypeByExtension("." + nodeExt),
- FirstCreated: creationTime,
- LastModified: fileInfo.ModTime(),
- Size: fileInfo.Size(),
- Misc: map[string]string{},
- }, nil
-}
-
-func (c *LocalUnixInputClient) ID(path string) string {
- return fmt.Sprintf("local-unix://%s%s", c.path, path)
-}
-
-func (c *LocalUnixInputClient) GetReader(path string) (io.ReadCloser, error) {
- return os.Open(c.path + path)
-}
diff --git a/internal/client/input/s3.go b/internal/client/input/s3.go
deleted file mode 100644
index 00684c2..0000000
--- a/internal/client/input/s3.go
+++ /dev/null
@@ -1,157 +0,0 @@
-package input
-
-import (
- "context"
- "fmt"
- "io"
- "slices"
- "strings"
-
- "github.com/aws/aws-sdk-go-v2/aws"
- awsconfig "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
-
- "github.com/SayaAndy/saya-today-thumbnail-generator/config"
-)
-
-var _ InputClient = (*S3InputClient)(nil)
-
-type S3InputClient struct {
- prefix string
- bucketName string
- s3cl *s3.Client
- knownExtensions []string
-}
-
-func NewS3InputClient(cfg *config.InputConfig) (InputClient, error) {
- if cfg.Storage.Type != "s3" {
- return nil, fmt.Errorf("invalid storage type for S3InputClient")
- }
- s3cfg := cfg.Storage.Config.(*config.S3Config)
-
- s3cl, err := newS3Client(s3cfg)
- if err != nil {
- return nil, fmt.Errorf("create S3 client: %w", err)
- }
-
- return &S3InputClient{
- s3cl: s3cl,
- bucketName: s3cfg.BucketName,
- prefix: s3cfg.Prefix,
- knownExtensions: cfg.KnownExtensions,
- }, nil
-}
-
-func (c *S3InputClient) Scan() ([]string, error) {
- var filePaths []string
-
- paginator := s3.NewListObjectsV2Paginator(c.s3cl, &s3.ListObjectsV2Input{
- Bucket: aws.String(c.bucketName),
- Prefix: aws.String(c.prefix),
- })
-
- for paginator.HasMorePages() {
- output, err := paginator.NextPage(context.Background())
- if err != nil {
- return nil, fmt.Errorf("list S3 objects: %w", err)
- }
-
- for _, obj := range output.Contents {
- name := aws.ToString(obj.Key)
-
- if len(c.knownExtensions) != 0 {
- nameParts := strings.Split(name, ".")
- if len(nameParts) < 2 {
- continue
- }
- ext := strings.ToLower(nameParts[len(nameParts)-1])
- if !slices.Contains(c.knownExtensions, ext) {
- continue
- }
- }
-
- filePaths = append(filePaths, strings.TrimPrefix(name, c.prefix))
- }
- }
-
- return filePaths, nil
-}
-
-func (c *S3InputClient) ReadMetadata(path string) (*MetadataStruct, error) {
- key := c.prefix + path
-
- head, err := c.s3cl.HeadObject(context.Background(), &s3.HeadObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("head S3 object %s: %w", key, err)
- }
-
- metadata := MetadataStruct{
- Name: key,
- StorageType: "s3",
- Hash: strings.Trim(aws.ToString(head.ETag), "\""),
- ContentType: aws.ToString(head.ContentType),
- Misc: head.Metadata,
- }
-
- if head.ContentLength != nil {
- metadata.Size = *head.ContentLength
- }
- if head.LastModified != nil {
- metadata.LastModified = *head.LastModified
- metadata.FirstCreated = *head.LastModified
- }
-
- return &metadata, nil
-}
-
-func (c *S3InputClient) ID(path string) string {
- return fmt.Sprintf("s3://%s/%s%s", c.bucketName, c.prefix, path)
-}
-
-func (c *S3InputClient) GetReader(path string) (io.ReadCloser, error) {
- key := c.prefix + path
-
- output, err := c.s3cl.GetObject(context.Background(), &s3.GetObjectInput{
- Bucket: aws.String(c.bucketName),
- Key: aws.String(key),
- })
- if err != nil {
- return nil, fmt.Errorf("get S3 object %s: %w", key, err)
- }
-
- return output.Body, nil
-}
-
-func newS3Client(cfg *config.S3Config) (*s3.Client, error) {
- opts := []func(*awsconfig.LoadOptions) error{
- awsconfig.WithRegion(cfg.Region),
- }
-
- if cfg.AccessKeyID != "" && cfg.SecretAccessKey != "" {
- opts = append(opts, awsconfig.WithCredentialsProvider(
- credentials.NewStaticCredentialsProvider(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
- ))
- }
-
- awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...)
- if err != nil {
- return nil, fmt.Errorf("load AWS config: %w", err)
- }
-
- var s3Opts []func(*s3.Options)
- if cfg.Endpoint != "" {
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.BaseEndpoint = aws.String(cfg.Endpoint)
- })
- }
- s3Opts = append(s3Opts, func(o *s3.Options) {
- o.UsePathStyle = cfg.UsePathStyle
- o.DisableLogOutputChecksumValidationSkipped = true
- })
-
- return s3.NewFromConfig(awsCfg, s3Opts...), nil
-}
generated by cgitext v1.3.1+17 (git 2.54.0) at 2026-09-10 20:51:21 +0000