summaryrefslogtreecommitdiff
diff options
from:
to:
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore2
-rw-r--r--config/config.go14
-rw-r--r--config/config.json8
-rw-r--r--internal/storage/b2.go75
-rw-r--r--internal/storage/storage_interface.go4
-rw-r--r--main.go24
6 files changed, 86 insertions, 41 deletions
diff --git a/.gitignore b/.gitignore
index ed20272..a059579 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,5 @@ go.work.sum
# Editor/IDE
.idea/
.vscode/
+
+.envrc
diff --git a/config/config.go b/config/config.go
index 515047e..b41be13 100644
--- a/config/config.go
+++ b/config/config.go
@@ -10,9 +10,17 @@ import (
)
type Config struct {
- LogLevel slog.Level `json:"LogLevel" validate:"required"`
- Storage StorageConfig `json:"Storage" validate:"required"`
- MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"`
+ LogLevel slog.Level `json:"LogLevel" validate:"required"`
+ MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"`
+ DraftMode DraftModeConfig `json:"DraftMode"`
+ Storage StorageConfig `json:"Storage" validate:"required"`
+}
+
+type DraftModeConfig struct {
+ Enabled bool `json:"Enabled" validate:"required"`
+ DraftSuffix string `json:"DraftSuffix" validate:"required_if=Enabled true"`
+ ProdSuffix string `json:"ProdSuffix" validate:"required_if=Enabled true"`
+ UpdateAlways bool `json:"UpdateAlways"`
}
type StorageConfig struct {
diff --git a/config/config.json b/config/config.json
index a48f0ff..985dd55 100644
--- a/config/config.json
+++ b/config/config.json
@@ -1,6 +1,12 @@
{
"LogLevel": "debug",
- "MaxConcurrentJobs": 8,
+ "MaxConcurrentJobs": 10,
+ "DraftMode": {
+ "Enabled": true,
+ "DraftSuffix": ".draft.md",
+ "ProdSuffix": ".md",
+ "UpdateAlways": false
+ },
"Storage": {
"Type": "b2",
"Config": {
diff --git a/internal/storage/b2.go b/internal/storage/b2.go
index 0df321f..7d7ad0b 100644
--- a/internal/storage/b2.go
+++ b/internal/storage/b2.go
@@ -16,12 +16,13 @@ import (
var _ StorageClient = &B2StorageClient{}
type B2StorageClient struct {
- prefix string
- bucket *b2.Bucket
- b2cl *b2.Client
+ prefix string
+ bucket *b2.Bucket
+ b2cl *b2.Client
+ draftModeCfg *config.DraftModeConfig
}
-func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
+func NewB2StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftModeConfig) (StorageClient, error) {
if cfg.Type != "b2" {
return nil, fmt.Errorf("invalid storage type for B2InputClient")
}
@@ -37,7 +38,9 @@ func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
return nil, err
}
- return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
+ draftModeCfgCopy := *draftModeCfg
+
+ return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, draftModeCfg: &draftModeCfgCopy}, nil
}
func (sc *B2StorageClient) Scan() ([]string, error) {
@@ -60,11 +63,16 @@ func (sc *B2StorageClient) Scan() ([]string, error) {
continue
}
- if !strings.HasSuffix(obj.Name(), ".md") {
+ name := obj.Name()
+ if !strings.HasSuffix(name, ".md") {
+ continue
+ }
+
+ if sc.draftModeCfg.Enabled && !strings.HasSuffix(name, sc.draftModeCfg.DraftSuffix) {
continue
}
- filePaths = append(filePaths, strings.TrimPrefix(obj.Name(), sc.prefix))
+ filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix))
}
if err := iter.Err(); err != nil {
@@ -88,13 +96,13 @@ func (sc *B2StorageClient) GetReader(path string) (io.ReadCloser, int64, error)
}
func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error {
- obj := sc.bucket.Object(sc.prefix + path)
- if obj == nil {
- return fmt.Errorf("failed to reference object in B2 bucket")
+ draft := sc.bucket.Object(sc.prefix + path)
+ if draft == nil {
+ return fmt.Errorf("failed to reference draft object in B2 bucket")
}
- oldAttrs, err := obj.Attrs(context.Background())
+ draftAttrs, err := draft.Attrs(context.Background())
if err != nil {
- return fmt.Errorf("error getting attributes of an object: %w", err)
+ return fmt.Errorf("error getting attributes of a draft object: %w", err)
}
geolocationParts := strings.Split(metadata.Geolocation, " ")
@@ -125,16 +133,27 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta
"thumbnail": metadata.Thumbnail,
"tags": strings.Join(metadata.Tags, ","),
"geolocation": metadata.Geolocation,
- "metadata-last-update-sha1": oldAttrs.SHA1,
+ "metadata-last-update-sha1": draftAttrs.SHA1,
}}
- reader := obj.NewReader(context.Background())
- content := make([]byte, oldAttrs.Size)
+ reader := draft.NewReader(context.Background())
+ content := make([]byte, draftAttrs.Size)
if _, err = reader.Read(content); err != nil {
- return fmt.Errorf("failed to read an object back for writing (required for attribute setting): %w", err)
+ return fmt.Errorf("failed to read a draft object back for writing (required for attribute setting): %w", err)
+ }
+
+ var prod *b2.Object
+ if sc.draftModeCfg.Enabled {
+ prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+ prod = sc.bucket.Object(sc.prefix + prodPath)
+ if prod == nil {
+ return fmt.Errorf("failed to reference prod object in B2 bucket")
+ }
+ } else {
+ prod = draft
}
- writer := obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs))
+ writer := prod.NewWriter(context.Background(), b2.WithAttrsOption(attrs))
defer writer.Close()
if _, err := writer.Write(content); err != nil {
return fmt.Errorf("failed to write an object back after attribute settings: %w", err)
@@ -143,21 +162,31 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta
return nil
}
-func (sc *B2StorageClient) FileHasChanged(path string) bool {
- obj := sc.bucket.Object(sc.prefix + path)
- if obj == nil {
+func (sc *B2StorageClient) CompareDraftAndProd(path string) (changed bool) {
+ draft := sc.bucket.Object(sc.prefix + path)
+ if draft == nil {
+ return false
+ }
+
+ prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+ prod := sc.bucket.Object(sc.prefix + prodPath)
+ if prod == nil {
return true
}
- attrs, err := obj.Attrs(context.Background())
+ draftAttrs, err := draft.Attrs(context.Background())
+ if err != nil {
+ return false
+ }
+ prodAttrs, err := prod.Attrs(context.Background())
if err != nil {
return true
}
- lastUpdateSha1, ok := attrs.Info["metadata-last-update-sha1"]
+ lastUpdateSha1, ok := prodAttrs.Info["metadata-last-update-sha1"]
if !ok {
return true
}
- return attrs.SHA1 == lastUpdateSha1
+ return draftAttrs.SHA1 != lastUpdateSha1
}
diff --git a/internal/storage/storage_interface.go b/internal/storage/storage_interface.go
index 91eeedb..02e0790 100644
--- a/internal/storage/storage_interface.go
+++ b/internal/storage/storage_interface.go
@@ -11,9 +11,9 @@ type StorageClient interface {
Scan() (paths []string, err error)
GetReader(path string) (reader io.ReadCloser, sz int64, err error)
WriteMetadata(path string, metadata *frontmatter.Metadata) error
- FileHasChanged(path string) bool
+ CompareDraftAndProd(path string) (changed bool)
}
-var NewStorageClientMap = map[string]func(cfg *config.StorageConfig) (StorageClient, error){
+var NewStorageClientMap = map[string]func(*config.StorageConfig, *config.DraftModeConfig) (StorageClient, error){
"b2": NewB2StorageClient,
}
diff --git a/main.go b/main.go
index 7c50824..04b0a4d 100644
--- a/main.go
+++ b/main.go
@@ -25,7 +25,7 @@ func main() {
slog.SetLogLoggerLevel(cfg.LogLevel)
slog.Info("starting metadata extractor...")
- storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage)
+ storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage, &cfg.DraftMode)
if err != nil {
slog.Error("fail to initialize input client", slog.String("error", err.Error()))
os.Exit(1)
@@ -52,15 +52,15 @@ func main() {
go func(index int, inputName string) {
defer wg.Done()
defer func() { <-semaphore }()
- if !storageClient.FileHasChanged(file) {
- generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file))
+ if cfg.DraftMode.Enabled && !storageClient.CompareDraftAndProd(inputName) {
+ generalLogger.Debug("skip a draft because prod object is identical to it", slog.String("file", inputName))
return
}
- generalLogger.Debug("processing a file", slog.String("file", file))
+ generalLogger.Debug("processing a file", slog.String("file", inputName))
- reader, sz, err := storageClient.GetReader(file)
+ reader, sz, err := storageClient.GetReader(inputName)
if err != nil {
- generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error()))
+ generalLogger.Warn("fail to get reader for a file", slog.String("file", inputName), slog.String("error", err.Error()))
return
}
defer reader.Close()
@@ -68,27 +68,27 @@ func main() {
content := make([]byte, sz)
ln, err := reader.Read(content)
if err != nil {
- generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error()))
+ generalLogger.Warn("fail to read content from a file", slog.String("file", inputName), slog.String("error", err.Error()))
return
}
generalLogger.Debug("read content from a file",
- slog.String("file", file),
+ slog.String("file", inputName),
slog.Int64("expected_size", sz),
slog.Int("output_size", ln))
metadata, _, err := frontmatter.ParseFrontmatter(content)
if err != nil {
- generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error()))
+ generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", inputName), slog.String("error", err.Error()))
return
}
if metadata == nil {
- generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file))
+ generalLogger.Info("skip a file due to it not having metadata", slog.String("file", inputName))
return
}
- if err = storageClient.WriteMetadata(file, metadata); err != nil {
- generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error()))
+ if err = storageClient.WriteMetadata(inputName, metadata); err != nil {
+ generalLogger.Warn("fail to write metadata to a file", slog.String("file", inputName), slog.String("error", err.Error()))
}
}(i, file)
}
generated by cgitext v1.3.1+13 (git 2.54.0) at 2026-08-20 11:24:34 +0000