diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | config/config.go | 83 | ||||
| -rw-r--r-- | config/config.json | 13 | ||||
| -rw-r--r-- | go.mod | 17 | ||||
| -rw-r--r-- | go.sum | 23 | ||||
| -rw-r--r-- | internal/frontmatter/parser.go | 37 | ||||
| -rw-r--r-- | internal/storage/b2.go | 140 | ||||
| -rw-r--r-- | internal/storage/storage_interface.go | 19 | ||||
| -rw-r--r-- | main.go | 85 |
9 files changed, 419 insertions, 2 deletions
@@ -28,5 +28,5 @@ go.work.sum .env # Editor/IDE -# .idea/ -# .vscode/ +.idea/ +.vscode/ diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..63c1e0c --- /dev/null +++ b/config/config.go @@ -0,0 +1,83 @@ +package config + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + + "github.com/go-playground/validator/v10" +) + +type Config struct { + LogLevel slog.Level `json:"LogLevel" validate:"required"` + Storage StorageConfig `json:"Storage" validate:"required"` +} + +type StorageConfig struct { + Type string `json:"Type" validate:"required,oneof=b2"` + Config any `json:"Config" validate:"required"` +} + +func (sc *StorageConfig) UnmarshalJSON(data []byte) error { + var tmp struct { + Type string `json:"Type"` + Config json.RawMessage `json:"Config"` + } + + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + sc.Type = tmp.Type + + switch tmp.Type { + case "b2": + var b2Config B2Config + if err := json.Unmarshal(tmp.Config, &b2Config); err != nil { + return fmt.Errorf("unmarshal B2Config: %w", err) + } + sc.Config = &b2Config + default: + return fmt.Errorf("unsupported storage type: %s", tmp.Type) + } + + return nil +} + +type B2Config struct { + BucketName string `json:"BucketName" validate:"required,min=1"` + Region string `json:"Region" validate:"required,min=1"` + Prefix string `json:"Prefix"` + KeyID string `json:"KeyID"` + ApplicationKey string `json:"ApplicationKey"` +} + +func LoadConfig(path string, config *Config) error { + fileBytes, err := os.ReadFile(path) + if err != nil { + return err + } + + expandedFileBytes := []byte(os.ExpandEnv(string(fileBytes))) + + if err = json.Unmarshal(expandedFileBytes, config); err != nil { + return err + } + + return nil +} + +func InitConfig(path string) (*Config, error) { + config := &Config{} + if err := LoadConfig(path, config); err != nil { + return nil, err + } + + validate := validator.New(validator.WithRequiredStructEnabled()) + if err := validate.Struct(config); err != nil { + return nil, err + } + + return config, nil +} diff --git a/config/config.json b/config/config.json new file mode 100644 index 0000000..9a49b76 --- /dev/null +++ b/config/config.json @@ -0,0 +1,13 @@ +{ + "LogLevel": "debug", + "Storage": { + "Type": "b2", + "Config": { + "BucketName": "sayana-pages", + "Region": "eu-central-003", + "Prefix": "", + "KeyID": "${B2_KEY_ID}", + "ApplicationKey": "${B2_APPLICATION_KEY}" + } + } +}
\ No newline at end of file @@ -0,0 +1,17 @@ +module github.com/SayaAndy/saya-today-article-metadata-add + +go 1.24.5 + +require ( + github.com/Backblaze/blazer v0.7.2 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) @@ -0,0 +1,23 @@ +github.com/Backblaze/blazer v0.7.2 h1:UWNHMLB+Nf+UmbO2qkVvgriODLEMz4kIyr2Hm+DVXQM= +github.com/Backblaze/blazer v0.7.2/go.mod h1:T4y3EYa9IQ5J0PKc/C/J8/CEnSd3qa/lgNw938wZg10= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/frontmatter/parser.go b/internal/frontmatter/parser.go new file mode 100644 index 0000000..6cc3ee8 --- /dev/null +++ b/internal/frontmatter/parser.go @@ -0,0 +1,37 @@ +package frontmatter + +import ( + "fmt" + "regexp" + "time" + + "gopkg.in/yaml.v3" +) + +type Metadata struct { + Title string `yaml:"title"` + ShortDescription string `yaml:"shortDescription"` + ActionDate string `yaml:"actionDate"` + PublishedTime time.Time `yaml:"publishedTime"` + Thumbnail string `yaml:"thumbnail"` + Tags []string `yaml:"tags"` +} + +func ParseFrontmatter(content []byte) (metadata *Metadata, markdown []byte, err error) { + frontmatterRegex := regexp.MustCompile(`^---\s*\r?\n([\s\S]*?)\r?\n---\s*\r?\n([\s\S]*)$`) + matches := frontmatterRegex.FindSubmatch(content) + + if len(matches) != 3 { + return nil, content, nil + } + + yamlContent := matches[1] + markdownContent := matches[2] + + metadata = &Metadata{} + if err := yaml.Unmarshal([]byte(yamlContent), &metadata); err != nil { + return nil, nil, fmt.Errorf("failed to parse YAML frontmatter: %w", err) + } + + return metadata, markdownContent, nil +} diff --git a/internal/storage/b2.go b/internal/storage/b2.go new file mode 100644 index 0000000..168690a --- /dev/null +++ b/internal/storage/b2.go @@ -0,0 +1,140 @@ +package storage + +import ( + "context" + "fmt" + "io" + "strings" + "time" + + "github.com/Backblaze/blazer/b2" + "github.com/SayaAndy/saya-today-article-metadata-add/config" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" +) + +var _ StorageClient = &B2StorageClient{} + +type B2StorageClient struct { + prefix string + bucket *b2.Bucket + b2cl *b2.Client +} + +func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) { + if cfg.Type != "b2" { + return nil, fmt.Errorf("invalid storage type for B2InputClient") + } + b2cfg := cfg.Config.(*config.B2Config) + + b2cl, err := b2.NewClient(context.Background(), b2cfg.KeyID, b2cfg.ApplicationKey) + if err != nil { + return nil, err + } + + bucket, err := b2cl.Bucket(context.Background(), b2cfg.BucketName) + if err != nil { + return nil, err + } + + return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil +} + +func (sc *B2StorageClient) Scan() ([]string, error) { + filePaths := []string{} + + iter := sc.bucket.List(context.Background(), b2.ListPrefix(sc.prefix)) + + for iter.Next() { + obj := iter.Object() + if obj == nil { + return nil, fmt.Errorf("failed to reference object in B2 bucket") + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, fmt.Errorf("get attributes for object: %w", err) + } + + if attrs.Status != b2.Uploaded { + continue + } + + name := obj.Name() + + nameParts := strings.Split(name, ".") + if len(nameParts) < 2 { + continue + } + + ext := strings.ToLower(nameParts[len(nameParts)-1]) + if ext != "md" { + continue + } + + filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix)) + } + + if err := iter.Err(); err != nil { + return nil, fmt.Errorf("iterate over B2 objects: %w", err) + } + + return filePaths, nil +} + +func (sc *B2StorageClient) GetReader(path string) (io.ReadCloser, int64, error) { + obj := sc.bucket.Object(sc.prefix + path) + if obj == nil { + return nil, 0, fmt.Errorf("failed to reference object in B2 bucket") + } + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return nil, 0, fmt.Errorf("error getting attributes of an object: %w", err) + } + + return obj.NewReader(context.Background()), attrs.Size, nil +} + +func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error { + obj := sc.bucket.Object(sc.prefix + path) + if obj == nil { + return fmt.Errorf("failed to reference object in B2 bucket") + } + oldAttrs, err := obj.Attrs(context.Background()) + if err != nil { + return fmt.Errorf("error getting attributes of an object: %w", err) + } + + attrs := &b2.Attrs{Info: map[string]string{ + "title": metadata.Title, + "short-description": metadata.ShortDescription, + "action-date": metadata.ActionDate, + "published-time": metadata.PublishedTime.Format(time.RFC3339), + "thumbnail": metadata.Thumbnail, + "tags": strings.Join(metadata.Tags, ","), + "metadata-last-update-sha1": oldAttrs.SHA1, + }} + + writer := obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs)) + writer.Close() + + return nil +} + +func (sc *B2StorageClient) FileHasChanged(path string) bool { + obj := sc.bucket.Object(sc.prefix + path) + if obj == nil { + return true + } + + attrs, err := obj.Attrs(context.Background()) + if err != nil { + return true + } + + lastUpdateSha1, ok := attrs.Info["metadata-last-update-sha1"] + if !ok { + return true + } + + return attrs.SHA1 == lastUpdateSha1 +} diff --git a/internal/storage/storage_interface.go b/internal/storage/storage_interface.go new file mode 100644 index 0000000..91eeedb --- /dev/null +++ b/internal/storage/storage_interface.go @@ -0,0 +1,19 @@ +package storage + +import ( + "io" + + "github.com/SayaAndy/saya-today-article-metadata-add/config" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" +) + +type StorageClient interface { + Scan() (paths []string, err error) + GetReader(path string) (reader io.ReadCloser, sz int64, err error) + WriteMetadata(path string, metadata *frontmatter.Metadata) error + FileHasChanged(path string) bool +} + +var NewStorageClientMap = map[string]func(cfg *config.StorageConfig) (StorageClient, error){ + "b2": NewB2StorageClient, +} @@ -0,0 +1,85 @@ +package main + +import ( + "flag" + "log/slog" + "os" + + "github.com/SayaAndy/saya-today-article-metadata-add/config" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" + "github.com/SayaAndy/saya-today-article-metadata-add/internal/storage" +) + +var configPath = flag.String("c", "config.json", "Path to the configuration file") + +func main() { + flag.Parse() + + cfg := &config.Config{} + if err := config.LoadConfig(*configPath, cfg); err != nil { + slog.Error("fail to load configuration", slog.String("error", err.Error())) + os.Exit(1) + } + + slog.SetLogLoggerLevel(cfg.LogLevel) + slog.Info("starting metadata extractor...") + + storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage) + if err != nil { + slog.Error("fail to initialize input client", slog.String("error", err.Error())) + os.Exit(1) + } + + generalLogger := slog.With( + slog.String("storage_type", cfg.Storage.Type), + ) + generalLogger.Info("initialized storage client") + + files, err := storageClient.Scan() + if err != nil { + generalLogger.Error("fail to scan input files", slog.String("error", err.Error())) + os.Exit(1) + } + generalLogger.Info("scanned files", slog.Int("file_count", len(files))) + + for _, file := range files { + if !storageClient.FileHasChanged(file) { + generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file)) + continue + } + generalLogger.Debug("processing a file", slog.String("file", file)) + + reader, sz, err := storageClient.GetReader(file) + if err != nil { + generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error())) + continue + } + defer reader.Close() + + content := make([]byte, sz) + ln, err := reader.Read(content) + if err != nil { + generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error())) + continue + } + generalLogger.Debug("read content from a file", + slog.String("file", file), + slog.Int64("expected_size", sz), + slog.Int("output_size", ln)) + + metadata, _, err := frontmatter.ParseFrontmatter(content) + if err != nil { + generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error())) + continue + } + + if metadata == nil { + generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file)) + continue + } + + if err = storageClient.WriteMetadata(file, metadata); err != nil { + generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error())) + } + } +} |