feat: new draft and prod mode

6 files changed, 86 insertions, 41 deletions

diff --git a/.gitignore b/.gitignore
index ed20272..a059579 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,5 @@ go.work.sum
 # Editor/IDE
 .idea/
 .vscode/
+
+.envrc
diff --git a/config/config.go b/config/config.go
index 515047e..b41be13 100644
--- a/config/config.go
+++ b/config/config.go
@@ -10,9 +10,17 @@ import (
 )
 
 type Config struct {
-	LogLevel          slog.Level    `json:"LogLevel" validate:"required"`
-	Storage           StorageConfig `json:"Storage" validate:"required"`
-	MaxConcurrentJobs int           `json:"MaxConcurrentJobs" validate:"required,min=1"`
+	LogLevel          slog.Level      `json:"LogLevel" validate:"required"`
+	MaxConcurrentJobs int             `json:"MaxConcurrentJobs" validate:"required,min=1"`
+	DraftMode         DraftModeConfig `json:"DraftMode"`
+	Storage           StorageConfig   `json:"Storage" validate:"required"`
+}
+
+type DraftModeConfig struct {
+	Enabled      bool   `json:"Enabled" validate:"required"`
+	DraftSuffix  string `json:"DraftSuffix" validate:"required_if=Enabled true"`
+	ProdSuffix   string `json:"ProdSuffix" validate:"required_if=Enabled true"`
+	UpdateAlways bool   `json:"UpdateAlways"`
 }
 
 type StorageConfig struct {
diff --git a/config/config.json b/config/config.json
index a48f0ff..985dd55 100644
--- a/config/config.json
+++ b/config/config.json
@@ -1,6 +1,12 @@
 {
     "LogLevel": "debug",
-    "MaxConcurrentJobs": 8,
+    "MaxConcurrentJobs": 10,
+    "DraftMode": {
+        "Enabled": true,
+        "DraftSuffix": ".draft.md",
+        "ProdSuffix": ".md",
+        "UpdateAlways": false
+    },
     "Storage": {
         "Type": "b2",
         "Config": {
diff --git a/internal/storage/b2.go b/internal/storage/b2.go
index 0df321f..7d7ad0b 100644
--- a/internal/storage/b2.go
+++ b/internal/storage/b2.go
@@ -16,12 +16,13 @@ import (
 var _ StorageClient = &B2StorageClient{}
 
 type B2StorageClient struct {
-	prefix string
-	bucket *b2.Bucket
-	b2cl   *b2.Client
+	prefix       string
+	bucket       *b2.Bucket
+	b2cl         *b2.Client
+	draftModeCfg *config.DraftModeConfig
 }
 
-func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
+func NewB2StorageClient(cfg *config.StorageConfig, draftModeCfg *config.DraftModeConfig) (StorageClient, error) {
 	if cfg.Type != "b2" {
 		return nil, fmt.Errorf("invalid storage type for B2InputClient")
 	}
@@ -37,7 +38,9 @@ func NewB2StorageClient(cfg *config.StorageConfig) (StorageClient, error) {
 		return nil, err
 	}
 
-	return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix}, nil
+	draftModeCfgCopy := *draftModeCfg
+
+	return &B2StorageClient{b2cl: b2cl, bucket: bucket, prefix: b2cfg.Prefix, draftModeCfg: &draftModeCfgCopy}, nil
 }
 
 func (sc *B2StorageClient) Scan() ([]string, error) {
@@ -60,11 +63,16 @@ func (sc *B2StorageClient) Scan() ([]string, error) {
 			continue
 		}
 
-		if !strings.HasSuffix(obj.Name(), ".md") {
+		name := obj.Name()
+		if !strings.HasSuffix(name, ".md") {
+			continue
+		}
+
+		if sc.draftModeCfg.Enabled && !strings.HasSuffix(name, sc.draftModeCfg.DraftSuffix) {
 			continue
 		}
 
-		filePaths = append(filePaths, strings.TrimPrefix(obj.Name(), sc.prefix))
+		filePaths = append(filePaths, strings.TrimPrefix(name, sc.prefix))
 	}
 
 	if err := iter.Err(); err != nil {
@@ -88,13 +96,13 @@ func (sc *B2StorageClient) GetReader(path string) (io.ReadCloser, int64, error)
 }
 
 func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Metadata) error {
-	obj := sc.bucket.Object(sc.prefix + path)
-	if obj == nil {
-		return fmt.Errorf("failed to reference object in B2 bucket")
+	draft := sc.bucket.Object(sc.prefix + path)
+	if draft == nil {
+		return fmt.Errorf("failed to reference draft object in B2 bucket")
 	}
-	oldAttrs, err := obj.Attrs(context.Background())
+	draftAttrs, err := draft.Attrs(context.Background())
 	if err != nil {
-		return fmt.Errorf("error getting attributes of an object: %w", err)
+		return fmt.Errorf("error getting attributes of a draft object: %w", err)
 	}
 
 	geolocationParts := strings.Split(metadata.Geolocation, " ")
@@ -125,16 +133,27 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta
 			"thumbnail":                 metadata.Thumbnail,
 			"tags":                      strings.Join(metadata.Tags, ","),
 			"geolocation":               metadata.Geolocation,
-			"metadata-last-update-sha1": oldAttrs.SHA1,
+			"metadata-last-update-sha1": draftAttrs.SHA1,
 		}}
 
-	reader := obj.NewReader(context.Background())
-	content := make([]byte, oldAttrs.Size)
+	reader := draft.NewReader(context.Background())
+	content := make([]byte, draftAttrs.Size)
 	if _, err = reader.Read(content); err != nil {
-		return fmt.Errorf("failed to read an object back for writing (required for attribute setting): %w", err)
+		return fmt.Errorf("failed to read a draft object back for writing (required for attribute setting): %w", err)
+	}
+
+	var prod *b2.Object
+	if sc.draftModeCfg.Enabled {
+		prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+		prod = sc.bucket.Object(sc.prefix + prodPath)
+		if prod == nil {
+			return fmt.Errorf("failed to reference prod object in B2 bucket")
+		}
+	} else {
+		prod = draft
 	}
 
-	writer := obj.NewWriter(context.Background(), b2.WithAttrsOption(attrs))
+	writer := prod.NewWriter(context.Background(), b2.WithAttrsOption(attrs))
 	defer writer.Close()
 	if _, err := writer.Write(content); err != nil {
 		return fmt.Errorf("failed to write an object back after attribute settings: %w", err)
@@ -143,21 +162,31 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta
 	return nil
 }
 
-func (sc *B2StorageClient) FileHasChanged(path string) bool {
-	obj := sc.bucket.Object(sc.prefix + path)
-	if obj == nil {
+func (sc *B2StorageClient) CompareDraftAndProd(path string) (changed bool) {
+	draft := sc.bucket.Object(sc.prefix + path)
+	if draft == nil {
+		return false
+	}
+
+	prodPath := strings.TrimSuffix(path, sc.draftModeCfg.DraftSuffix) + sc.draftModeCfg.ProdSuffix
+	prod := sc.bucket.Object(sc.prefix + prodPath)
+	if prod == nil {
 		return true
 	}
 
-	attrs, err := obj.Attrs(context.Background())
+	draftAttrs, err := draft.Attrs(context.Background())
+	if err != nil {
+		return false
+	}
+	prodAttrs, err := prod.Attrs(context.Background())
 	if err != nil {
 		return true
 	}
 
-	lastUpdateSha1, ok := attrs.Info["metadata-last-update-sha1"]
+	lastUpdateSha1, ok := prodAttrs.Info["metadata-last-update-sha1"]
 	if !ok {
 		return true
 	}
 
-	return attrs.SHA1 == lastUpdateSha1
+	return draftAttrs.SHA1 != lastUpdateSha1
 }
diff --git a/internal/storage/storage_interface.go b/internal/storage/storage_interface.go
index 91eeedb..02e0790 100644
--- a/internal/storage/storage_interface.go
+++ b/internal/storage/storage_interface.go
@@ -11,9 +11,9 @@ type StorageClient interface {
 	Scan() (paths []string, err error)
 	GetReader(path string) (reader io.ReadCloser, sz int64, err error)
 	WriteMetadata(path string, metadata *frontmatter.Metadata) error
-	FileHasChanged(path string) bool
+	CompareDraftAndProd(path string) (changed bool)
 }
 
-var NewStorageClientMap = map[string]func(cfg *config.StorageConfig) (StorageClient, error){
+var NewStorageClientMap = map[string]func(*config.StorageConfig, *config.DraftModeConfig) (StorageClient, error){
 	"b2": NewB2StorageClient,
 }
diff --git a/main.go b/main.go
index 7c50824..04b0a4d 100644
--- a/main.go
+++ b/main.go
@@ -25,7 +25,7 @@ func main() {
 	slog.SetLogLoggerLevel(cfg.LogLevel)
 	slog.Info("starting metadata extractor...")
 
-	storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage)
+	storageClient, err := storage.NewStorageClientMap[cfg.Storage.Type](&cfg.Storage, &cfg.DraftMode)
 	if err != nil {
 		slog.Error("fail to initialize input client", slog.String("error", err.Error()))
 		os.Exit(1)
@@ -52,15 +52,15 @@ func main() {
 		go func(index int, inputName string) {
 			defer wg.Done()
 			defer func() { <-semaphore }()
-			if !storageClient.FileHasChanged(file) {
-				generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file))
+			if cfg.DraftMode.Enabled && !storageClient.CompareDraftAndProd(inputName) {
+				generalLogger.Debug("skip a draft because prod object is identical to it", slog.String("file", inputName))
 				return
 			}
-			generalLogger.Debug("processing a file", slog.String("file", file))
+			generalLogger.Debug("processing a file", slog.String("file", inputName))
 
-			reader, sz, err := storageClient.GetReader(file)
+			reader, sz, err := storageClient.GetReader(inputName)
 			if err != nil {
-				generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error()))
+				generalLogger.Warn("fail to get reader for a file", slog.String("file", inputName), slog.String("error", err.Error()))
 				return
 			}
 			defer reader.Close()
@@ -68,27 +68,27 @@ func main() {
 			content := make([]byte, sz)
 			ln, err := reader.Read(content)
 			if err != nil {
-				generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error()))
+				generalLogger.Warn("fail to read content from a file", slog.String("file", inputName), slog.String("error", err.Error()))
 				return
 			}
 			generalLogger.Debug("read content from a file",
-				slog.String("file", file),
+				slog.String("file", inputName),
 				slog.Int64("expected_size", sz),
 				slog.Int("output_size", ln))
 
 			metadata, _, err := frontmatter.ParseFrontmatter(content)
 			if err != nil {
-				generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error()))
+				generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", inputName), slog.String("error", err.Error()))
 				return
 			}
 
 			if metadata == nil {
-				generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file))
+				generalLogger.Info("skip a file due to it not having metadata", slog.String("file", inputName))
 				return
 			}
 
-			if err = storageClient.WriteMetadata(file, metadata); err != nil {
-				generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error()))
+			if err = storageClient.WriteMetadata(inputName, metadata); err != nil {
+				generalLogger.Warn("fail to write metadata to a file", slog.String("file", inputName), slog.String("error", err.Error()))
 			}
 		}(i, file)
 	}