diff options
Diffstat (limited to 'internal/storage')
| -rw-r--r-- | internal/storage/s3.go | 224 | ||||
| -rw-r--r-- | internal/storage/storage_interface.go | 1 |
2 files changed, 225 insertions, 0 deletions
diff --git a/internal/storage/s3.go b/internal/storage/s3.go new file mode 100644 index 0000000..5e9bcd2 --- /dev/null +++ b/internal/storage/s3.go @@ -0,0 +1,224 @@ +package storage + +import ( + "bytes" + "context" + "fmt" + "io" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/SayaAndy/saya-today-article-metadata-add/config" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" +) + +var _ StorageClient = &S3StorageClient{} + +type S3StorageClient struct { + prefix string + bucket string + client *s3.Client + draftModeCfg *config.DraftModeConfig +} + +func NewS3StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftModeConfig) (StorageClient, error) { + if cfg.Type != "s3" { + return nil, fmt.Errorf("invalid storage type for S3StorageClient") + } + s3cfg := cfg.Config.(*config.S3Config) + + opts := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithRegion(s3cfg.Region), + } + if s3cfg.AccessKeyID != "" && s3cfg.SecretAccessKey != "" { + opts = append(opts, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(s3cfg.AccessKeyID, s3cfg.SecretAccessKey, ""), + )) + } + + awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(), opts...) + if err != nil { + return nil, fmt.Errorf("load AWS config: %w", err) + } + + var s3Opts []func(*s3.Options) + if s3cfg.Endpoint != "" { + s3Opts = append(s3Opts, func(o *s3.Options) { + o.BaseEndpoint = aws.String(s3cfg.Endpoint) + }) + } + s3Opts = append(s3Opts, func(o *s3.Options) { + o.UsePathStyle = s3cfg.UsePathStyle + }) + + client := s3.NewFromConfig(awsCfg, s3Opts...) + + draftModeCfgCopy := *draftModeCfg + + return &S3StorageClient{ + client: client, + bucket: s3cfg.BucketName, + prefix: s3cfg.Prefix, + draftModeCfg: &draftModeCfgCopy, + }, nil +} + +func (sc *S3StorageClient) Scan() ([]string, error) { + var filePaths []string + + paginator := s3.NewListObjectsV2Paginator(sc.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(sc.bucket), + Prefix: aws.String(sc.prefix), + }) + + for paginator.HasMorePages() { + page, err := paginator.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("list S3 objects: %w", err) + } + + for _, obj := range page.Contents { + name := aws.ToString(obj.Key) + if !strings.HasSuffix(name, ".md") { + continue + } + + if sc.draftModeCfg.Enabled && !strings.HasSuffix(name, sc.draftModeCfg.DraftSuffix) { + continue + } + + filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix)) + } + } + + return filePaths, nil +} + +func (sc *S3StorageClient) GetReader(path string) (io.ReadCloser, int64, error) { + key := sc.prefix + path + + out, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, 0, fmt.Errorf("get S3 object %q: %w", key, err) + } + + return out.Body, aws.ToInt64(out.ContentLength), nil +} + +func (sc *S3StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error { + draftKey := sc.prefix + path + + getOut, err := sc.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(draftKey), + }) + if err != nil { + return fmt.Errorf("get draft object %q: %w", draftKey, err) + } + content, err := io.ReadAll(getOut.Body) + getOut.Body.Close() + if err != nil { + return fmt.Errorf("read draft object %q: %w", draftKey, err) + } + + headOut, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(draftKey), + }) + if err != nil { + return fmt.Errorf("head draft object %q: %w", draftKey, err) + } + draftETag := aws.ToString(headOut.ETag) + + geolocationParts := strings.Split(metadata.Geolocation, " ") + if (len(geolocationParts) == 1 && geolocationParts[0] != "") || len(geolocationParts) >= 4 { + return fmt.Errorf("invalid geolocation format, expecting '{x} {y} [areaError]' or an empty string") + } + if len(geolocationParts) >= 2 { + if _, err := strconv.ParseFloat(geolocationParts[0], 64); err != nil { + return fmt.Errorf("invalid geolocation parameter, expected float for X: %w", err) + } + if _, err := strconv.ParseFloat(geolocationParts[1], 64); err != nil { + return fmt.Errorf("invalid geolocation parameter, expected float for Y: %w", err) + } + } + if len(geolocationParts) == 3 { + if _, err := strconv.ParseFloat(geolocationParts[2], 64); err != nil { + return fmt.Errorf("invalid geolocation parameter, expected float for area error: %w", err) + } + } + + medley := "" + if metadata.Medley != "" { + medley = fmt.Sprintf("%s %d", metadata.Medley, metadata.MedleyPart) + } + + s3Metadata := map[string]string{ + "title": metadata.Title, + "short-description": metadata.ShortDescription, + "action-date": metadata.ActionDate, + "published-time": metadata.PublishedTime.Format(time.RFC3339), + "thumbnail": metadata.Thumbnail, + "tags": strings.Join(metadata.Tags, ","), + "geolocation": metadata.Geolocation, + "medley": medley, + "metadata-last-update-etag": draftETag, + } + + targetKey := draftKey + if sc.draftModeCfg.Enabled { + prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix + targetKey = sc.prefix + prodPath + } + + _, err = sc.client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(targetKey), + Body: bytes.NewReader(content), + ContentType: aws.String("text/markdown; charset=utf-8"), + Metadata: s3Metadata, + }) + if err != nil { + return fmt.Errorf("put S3 object %q: %w", targetKey, err) + } + + return nil +} + +func (sc *S3StorageClient) CompareDraftAndProd(path string) bool { + draftKey := sc.prefix + path + prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix + prodKey := sc.prefix + prodPath + + draftHead, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(draftKey), + }) + if err != nil { + return false + } + + prodHead, err := sc.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(sc.bucket), + Key: aws.String(prodKey), + }) + if err != nil { + return true + } + + lastUpdateETag, ok := prodHead.Metadata["metadata-last-update-etag"] + if !ok { + return true + } + + return aws.ToString(draftHead.ETag) != lastUpdateETag +} diff --git a/internal/storage/storage_interface.go b/internal/storage/storage_interface.go index 02e0790..7134e30 100644 --- a/internal/storage/storage_interface.go +++ b/internal/storage/storage_interface.go @@ -16,4 +16,5 @@ type StorageClient interface { var NewStorageClientMap = map[string]func(*config.StorageConfig, *config.DraftModeConfig) (StorageClient, error){ "b2": NewB2StorageClient, + "s3": NewS3StorageClient, } |