feat: integrate parallel execution

refactor: simplify frontmatter parser
feat: always check for '.md' extension
feat: new timezone field

5 files changed, 63 insertions, 45 deletions

diff --git a/config/config.go b/config/config.go
index 63c1e0c..515047e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -10,8 +10,9 @@ import (
 )
 
 type Config struct {
-	LogLevel slog.Level    `json:"LogLevel" validate:"required"`
-	Storage  StorageConfig `json:"Storage" validate:"required"`
+	LogLevel          slog.Level    `json:"LogLevel" validate:"required"`
+	Storage           StorageConfig `json:"Storage" validate:"required"`
+	MaxConcurrentJobs int           `json:"MaxConcurrentJobs" validate:"required,min=1"`
 }
 
 type StorageConfig struct {
diff --git a/config/config.json b/config/config.json
index 9a49b76..a48f0ff 100644
--- a/config/config.json
+++ b/config/config.json
@@ -1,5 +1,6 @@
 {
     "LogLevel": "debug",
+    "MaxConcurrentJobs": 8,
     "Storage": {
         "Type": "b2",
         "Config": {
diff --git a/internal/frontmatter/parser.go b/internal/frontmatter/parser.go
index 1a521c0..20512d9 100644
--- a/internal/frontmatter/parser.go
+++ b/internal/frontmatter/parser.go
@@ -1,8 +1,8 @@
 package frontmatter
 
 import (
+	"bytes"
 	"fmt"
-	"regexp"
 	"time"
 
 	"gopkg.in/yaml.v3"
@@ -16,18 +16,21 @@ type Metadata struct {
 	Thumbnail        string    `yaml:"thumbnail"`
 	Tags             []string  `yaml:"tags"`
 	Geolocation      string    `yaml:"geolocation"`
+	Timezone         string    `yaml:"timezone"`
 }
 
 func ParseFrontmatter(content []byte) (metadata *Metadata, markdown []byte, err error) {
-	frontmatterRegex := regexp.MustCompile(`^---\s*\r?\n([\s\S]*?)\r?\n---\s*\r?\n([\s\S]*)$`)
-	matches := frontmatterRegex.FindSubmatch(content)
+	if !bytes.HasPrefix(content, []byte("---\n")) {
+		return nil, content, nil
+	}
 
-	if len(matches) != 3 {
+	end := bytes.Index(content[4:], []byte("\n---\n"))
+	if end == -1 {
 		return nil, content, nil
 	}
 
-	yamlContent := matches[1]
-	markdownContent := matches[2]
+	yamlContent := content[4 : end+4]
+	markdownContent := content[end+9:]
 
 	metadata = &Metadata{}
 	if err := yaml.Unmarshal([]byte(yamlContent), &metadata); err != nil {
diff --git a/internal/storage/b2.go b/internal/storage/b2.go
index 0c4b673..0d92707 100644
--- a/internal/storage/b2.go
+++ b/internal/storage/b2.go
@@ -60,9 +60,9 @@ func (sc *B2StorageClient) Scan() ([]string, error) {
 			continue
 		}
 
-		//if !strings.Contains(attrs.ContentType, "text/metadata") {
-		//	continue
-		//}
+		if !strings.HasSuffix(obj.Name(), ".md") {
+			continue
+		}
 
 		filePaths = append(filePaths, strings.TrimPrefix(obj.Name(), sc.prefix))
 	}
@@ -125,6 +125,7 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta
 			"thumbnail":                 metadata.Thumbnail,
 			"tags":                      strings.Join(metadata.Tags, ","),
 			"geolocation":               metadata.Geolocation,
+			"timezone":                  metadata.Timezone,
 			"metadata-last-update-sha1": oldAttrs.SHA1,
 		}}
 
diff --git a/main.go b/main.go
index 4ff2f98..7c50824 100644
--- a/main.go
+++ b/main.go
@@ -4,6 +4,7 @@ import (
 	"flag"
 	"log/slog"
 	"os"
+	"sync"
 
 	"github.com/SayaAndy/saya-today-article-metadata-add/config"
 	"github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
@@ -42,44 +43,55 @@ func main() {
 	}
 	generalLogger.Info("scanned files", slog.Int("file_count", len(files)))
 
-	for _, file := range files {
-		if !storageClient.FileHasChanged(file) {
-			generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file))
-			continue
-		}
-		generalLogger.Debug("processing a file", slog.String("file", file))
+	semaphore := make(chan struct{}, cfg.MaxConcurrentJobs)
+	var wg sync.WaitGroup
+	wg.Add(len(files))
 
-		reader, sz, err := storageClient.GetReader(file)
-		if err != nil {
-			generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error()))
-			continue
-		}
-		defer reader.Close()
+	for i, file := range files {
+		semaphore <- struct{}{}
+		go func(index int, inputName string) {
+			defer wg.Done()
+			defer func() { <-semaphore }()
+			if !storageClient.FileHasChanged(file) {
+				generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file))
+				return
+			}
+			generalLogger.Debug("processing a file", slog.String("file", file))
 
-		content := make([]byte, sz)
-		ln, err := reader.Read(content)
-		if err != nil {
-			generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error()))
-			continue
-		}
-		generalLogger.Debug("read content from a file",
-			slog.String("file", file),
-			slog.Int64("expected_size", sz),
-			slog.Int("output_size", ln))
+			reader, sz, err := storageClient.GetReader(file)
+			if err != nil {
+				generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error()))
+				return
+			}
+			defer reader.Close()
 
-		metadata, _, err := frontmatter.ParseFrontmatter(content)
-		if err != nil {
-			generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error()))
-			continue
-		}
+			content := make([]byte, sz)
+			ln, err := reader.Read(content)
+			if err != nil {
+				generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error()))
+				return
+			}
+			generalLogger.Debug("read content from a file",
+				slog.String("file", file),
+				slog.Int64("expected_size", sz),
+				slog.Int("output_size", ln))
 
-		if metadata == nil {
-			generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file))
-			continue
-		}
+			metadata, _, err := frontmatter.ParseFrontmatter(content)
+			if err != nil {
+				generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error()))
+				return
+			}
 
-		if err = storageClient.WriteMetadata(file, metadata); err != nil {
-			generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error()))
-		}
+			if metadata == nil {
+				generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file))
+				return
+			}
+
+			if err = storageClient.WriteMetadata(file, metadata); err != nil {
+				generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error()))
+			}
+		}(i, file)
 	}
+
+	wg.Wait()
 }