| -rw-r--r-- | config/config.go | 5 | ||||
| -rw-r--r-- | config/config.json | 1 | ||||
| -rw-r--r-- | internal/frontmatter/parser.go | 15 | ||||
| -rw-r--r-- | internal/storage/b2.go | 7 | ||||
| -rw-r--r-- | main.go | 80 |
5 files changed, 45 insertions, 63 deletions
diff --git a/config/config.go b/config/config.go index 515047e..63c1e0c 100644 --- a/config/config.go +++ b/config/config.go @@ -10,9 +10,8 @@ import ( ) type Config struct { - LogLevel slog.Level `json:"LogLevel" validate:"required"` - Storage StorageConfig `json:"Storage" validate:"required"` - MaxConcurrentJobs int `json:"MaxConcurrentJobs" validate:"required,min=1"` + LogLevel slog.Level `json:"LogLevel" validate:"required"` + Storage StorageConfig `json:"Storage" validate:"required"` } type StorageConfig struct { diff --git a/config/config.json b/config/config.json index a48f0ff..9a49b76 100644 --- a/config/config.json +++ b/config/config.json @@ -1,6 +1,5 @@ { "LogLevel": "debug", - "MaxConcurrentJobs": 8, "Storage": { "Type": "b2", "Config": { diff --git a/internal/frontmatter/parser.go b/internal/frontmatter/parser.go index 20512d9..1a521c0 100644 --- a/internal/frontmatter/parser.go +++ b/internal/frontmatter/parser.go @@ -1,8 +1,8 @@ package frontmatter import ( - "bytes" "fmt" + "regexp" "time" "gopkg.in/yaml.v3" @@ -16,21 +16,18 @@ type Metadata struct { Thumbnail string `yaml:"thumbnail"` Tags []string `yaml:"tags"` Geolocation string `yaml:"geolocation"` - Timezone string `yaml:"timezone"` } func ParseFrontmatter(content []byte) (metadata *Metadata, markdown []byte, err error) { - if !bytes.HasPrefix(content, []byte("---\n")) { - return nil, content, nil - } + frontmatterRegex := regexp.MustCompile(`^---\s*\r?\n([\s\S]*?)\r?\n---\s*\r?\n([\s\S]*)$`) + matches := frontmatterRegex.FindSubmatch(content) - end := bytes.Index(content[4:], []byte("\n---\n")) - if end == -1 { + if len(matches) != 3 { return nil, content, nil } - yamlContent := content[4 : end+4] - markdownContent := content[end+9:] + yamlContent := matches[1] + markdownContent := matches[2] metadata = &Metadata{} if err := yaml.Unmarshal([]byte(yamlContent), &metadata); err != nil { diff --git a/internal/storage/b2.go b/internal/storage/b2.go index 0d92707..0c4b673 100644 --- a/internal/storage/b2.go +++ b/internal/storage/b2.go @@ -60,9 +60,9 @@ func (sc *B2StorageClient) Scan() ([]string, error) { continue } - if !strings.HasSuffix(obj.Name(), ".md") { - continue - } + //if !strings.Contains(attrs.ContentType, "text/metadata") { + // continue + //} filePaths = append(filePaths, strings.TrimPrefix(obj.Name(), sc.prefix)) } @@ -125,7 +125,6 @@ func (sc *B2StorageClient) WriteMetadata(path string, metadata *frontmatter.Meta "thumbnail": metadata.Thumbnail, "tags": strings.Join(metadata.Tags, ","), "geolocation": metadata.Geolocation, - "timezone": metadata.Timezone, "metadata-last-update-sha1": oldAttrs.SHA1, }} @@ -4,7 +4,6 @@ import ( "flag" "log/slog" "os" - "sync" "github.com/SayaAndy/saya-today-article-metadata-add/config" "github.com/SayaAndy/saya-today-article-metadata-add/internal/frontmatter" @@ -43,55 +42,44 @@ func main() { } generalLogger.Info("scanned files", slog.Int("file_count", len(files))) - semaphore := make(chan struct{}, cfg.MaxConcurrentJobs) - var wg sync.WaitGroup - wg.Add(len(files)) + for _, file := range files { + if !storageClient.FileHasChanged(file) { + generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file)) + continue + } + generalLogger.Debug("processing a file", slog.String("file", file)) - for i, file := range files { - semaphore <- struct{}{} - go func(index int, inputName string) { - defer wg.Done() - defer func() { <-semaphore }() - if !storageClient.FileHasChanged(file) { - generalLogger.Debug("skipped a file because it has not changed since last parse", slog.String("file", file)) - return - } - generalLogger.Debug("processing a file", slog.String("file", file)) + reader, sz, err := storageClient.GetReader(file) + if err != nil { + generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error())) + continue + } + defer reader.Close() - reader, sz, err := storageClient.GetReader(file) - if err != nil { - generalLogger.Warn("fail to get reader for a file", slog.String("file", file), slog.String("error", err.Error())) - return - } - defer reader.Close() + content := make([]byte, sz) + ln, err := reader.Read(content) + if err != nil { + generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error())) + continue + } + generalLogger.Debug("read content from a file", + slog.String("file", file), + slog.Int64("expected_size", sz), + slog.Int("output_size", ln)) - content := make([]byte, sz) - ln, err := reader.Read(content) - if err != nil { - generalLogger.Warn("fail to read content from a file", slog.String("file", file), slog.String("error", err.Error())) - return - } - generalLogger.Debug("read content from a file", - slog.String("file", file), - slog.Int64("expected_size", sz), - slog.Int("output_size", ln)) + metadata, _, err := frontmatter.ParseFrontmatter(content) + if err != nil { + generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error())) + continue + } - metadata, _, err := frontmatter.ParseFrontmatter(content) - if err != nil { - generalLogger.Warn("fail to parse frontmatter of a file", slog.String("file", file), slog.String("error", err.Error())) - return - } + if metadata == nil { + generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file)) + continue + } - if metadata == nil { - generalLogger.Info("skip a file due to it not having metadata", slog.String("file", file)) - return - } - - if err = storageClient.WriteMetadata(file, metadata); err != nil { - generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error())) - } - }(i, file) + if err = storageClient.WriteMetadata(file, metadata); err != nil { + generalLogger.Warn("fail to write metadata to a file", slog.String("file", file), slog.String("error", err.Error())) + } } - - wg.Wait() } |