summaryrefslogtreecommitdiff
diff options
from:
to:
context:
space:
mode:
Diffstat
-rw-r--r--config/config.go5
-rw-r--r--config/config.json1
-rw-r--r--internal/frontmatter/parser.go15
-rw-r--r--internal/storage/b2.go7
-rw-r--r--main.go80
5 files changed, 63 insertions, 45 deletions
diff --git a/config/config.go b/config/config.go
index 63c1e0c..515047e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -10,8 +10,9 @@ import (
)
type Config struct {
- LogLevel slog.Level `json:"LogLevel" validate:"required"`
- Storage StorageConfig `json:"Storage" validate:"required"`
+ LogLevel slog.Level `json:"LogLevel" validate:"required"`
+ Storage StorageConfig `json:"Storage" validate:"required"`
+ MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"`
}
type StorageConfig struct {
diff --git a/config/config.json b/config/config.json
index 9a49b76..a48f0ff 100644
--- a/config/config.json
+++ b/config/config.json
@@ -1,5 +1,6 @@
{
"LogLevel": "debug",
+ "MaxConcurrentJobs": 8,
"Storage": {
"Type": "b2",
"Config": {
diff --git a/internal/frontmatter/parser.go b/internal/frontmatter/parser.go
index 1a521c0..20512d9 100644
--- a/internal/frontmatter/parser.go
+++ b/internal/frontmatter/parser.go
@@ -1,8 +1,8 @@
package frontmatter
import (
+ "bytes"
"fmt"
- "regexp"
"time"
"gopkg.in/yaml.v3"
@@ -16,18 +16,21 @@ type Metadata struct {
Thumbnail string `yaml:"thumbnail"`
Tags []string `yaml:"tags"`
Geolocation string `yaml:"geolocation"`
+ Timezone string `yaml:"timezone"`
}
func ParseFrontmatter(content []byte) (metadata *Metadata, markdown []byte, err error) {
- frontmatterRegex := regexp.MustCompile(`^---\s*\r?\n([\s\S]*?)\r?\n---\s*\r?\n([\s\S]*)$`)
- matches := frontmatterRegex.FindSubmatch(content)
+ if !bytes.HasPrefix(content, []byte("---\n")) {
+ return nil, content, nil
+ }
- if len(matches) != 3 {
+ end := bytes.Index(content[4:], []byte("\n---\n"))
+ if end == -1 {
return nil, content, nil
}
- yamlContent := matches[1]
- markdownContent := matches[2]
+ yamlContent := content[4 : end+4]
+ markdownContent := content[end+9:]
metadata = &Metadata{}
if err := yaml.Unmarshal([]byte(yamlContent), &metadata); err != nil {
diff --git a/internal/storage/b2.go b/internal/storage/b2.go
index 0c4b673..0d92707 100644
--- a/internal/storage/b2.go
+++ b/internal/storage/b2.go
@@ -60,9 +60,9 @@ func (sc *B2StorageClient) Scan() ([]string, error) {
continue
}
- //if !strings.Contains(attrs.ContentType, "text/metadata") {
- // continue
- //}
+ if !strings.HasSuffix(obj.Name(), ".md") {
+ continue
+ }
filePaths = append(filePaths, strings.TrimPrefix(obj.Name(), sc.prefix))
}
@@ -125,6 +125,7 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta
"thumbnail": metadata.Thumbnail,
"tags": strings.Join(metadata.Tags, ","),
"geolocation": metadata.Geolocation,
+ "timezone": metadata.Timezone,
"metadata-last-update-sha1": oldAttrs.SHA1,
}}
diff --git a/main.go b/main.go
index 4ff2f98..7c50824 100644
--- a/main.go
+++ b/main.go
@@ -4,6 +4,7 @@ import (
"flag"
"log/slog"
"os"
+ "sync"
"github.com/SayaAndy/saya-today-article-metadata-add/config"
"github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter"
@@ -42,44 +43,55 @@ func main() {
}
generalLogger.Info("scanned files", slog.Int("file_count", len(files)))
- for _, file := range files {
- if !storageClient.FileHasChanged(file) {
- generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file))
- continue
- }
- generalLogger.Debug("processing a file", slog.String("file", file))
+ semaphore := make(chan struct{}, cfg.MaxConcurrentJobs)
+ var wg sync.WaitGroup
+ wg.Add(len(files))
- reader, sz, err := storageClient.GetReader(file)
- if err != nil {
- generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error()))
- continue
- }
- defer reader.Close()
+ for i, file := range files {
+ semaphore <- struct{}{}
+ go func(index int, inputName string) {
+ defer wg.Done()
+ defer func() { <-semaphore }()
+ if !storageClient.FileHasChanged(file) {
+ generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file))
+ return
+ }
+ generalLogger.Debug("processing a file", slog.String("file", file))
- content := make([]byte, sz)
- ln, err := reader.Read(content)
- if err != nil {
- generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error()))
- continue
- }
- generalLogger.Debug("read content from a file",
- slog.String("file", file),
- slog.Int64("expected_size", sz),
- slog.Int("output_size", ln))
+ reader, sz, err := storageClient.GetReader(file)
+ if err != nil {
+ generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error()))
+ return
+ }
+ defer reader.Close()
- metadata, _, err := frontmatter.ParseFrontmatter(content)
- if err != nil {
- generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error()))
- continue
- }
+ content := make([]byte, sz)
+ ln, err := reader.Read(content)
+ if err != nil {
+ generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error()))
+ return
+ }
+ generalLogger.Debug("read content from a file",
+ slog.String("file", file),
+ slog.Int64("expected_size", sz),
+ slog.Int("output_size", ln))
- if metadata == nil {
- generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file))
- continue
- }
+ metadata, _, err := frontmatter.ParseFrontmatter(content)
+ if err != nil {
+ generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error()))
+ return
+ }
- if err = storageClient.WriteMetadata(file, metadata); err != nil {
- generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error()))
- }
+ if metadata == nil {
+ generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file))
+ return
+ }
+
+ if err = storageClient.WriteMetadata(file, metadata); err != nil {
+ generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error()))
+ }
+ }(i, file)
}
+
+ wg.Wait()
}
generated by cgitext v1.3.1+13 (git 2.54.0) at 2026-08-20 11:17:17 +0000