1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
package main
import (
"flag"
"log/slog"
"sync"
"github.com/SayaAndy/saya-today-thumbnail-generator/config"
"github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/input"
"github.com/SayaAndy/saya-today-thumbnail-generator/internal/client/output"
"github.com/SayaAndy/saya-today-thumbnail-generator/internal/processor"
)
var (
configPath = flag.String("c", "config.json", "Path to the configuration file")
)
func main() {
flag.Parse()
slog.Info("starting thumbnail generator...")
// slog.SetLogLoggerLevel(slog.LevelDebug)
cfg := &config.Config{}
if err := config.LoadConfig(*configPath, cfg); err != nil {
panic(err)
}
inputClient, err := input.NewB2InputClient(&cfg.Input)
if err != nil {
panic(err)
}
outputClient, err := output.NewB2OutputClient(&cfg.Output)
if err != nil {
panic(err)
}
converter, err := processor.NewWebpProcessor(&cfg.Processor)
if err != nil {
panic(err)
}
generalLogger := slog.With(
slog.String("input_storage", cfg.Input.Storage.Type),
slog.String("processor_type", cfg.Processor.Type),
slog.String("output_storage", cfg.Output.Storage.Type),
)
generalLogger.Info("initialized clients and processor")
files, err := inputClient.Scan()
if err != nil {
panic(err)
}
generalLogger.Info("scanned files", slog.Int("file_count", len(files)))
semaphore := make(chan struct{}, cfg.MaxConcurrentJobs)
var wg sync.WaitGroup
wg.Add(len(files))
for i, file := range files {
go func(index int, inputName string) {
semaphore <- struct{}{}
defer func() { <-semaphore }()
defer wg.Done()
outputName := converter.DeductOutputPath(inputName)
fileLogger := generalLogger.With(slog.String("input_path", inputName), slog.String("output_path", outputName), slog.Int("file_index", index))
inputMetadata, err := inputClient.ReadMetadata(inputName)
if err != nil {
fileLogger.Error("fail to read metadata of (supposedly existing) input file", slog.String("error", err.Error()))
return
}
if !cfg.ForceRewrite && !outputClient.IsMissing(outputName) {
outputMetadata, err := outputClient.ReadMetadata(outputName)
if err != nil {
fileLogger.Error("fail to read metadata of (supposedly existing) output file", slog.String("error", err.Error()))
return
}
if inputMetadata.Hash == outputMetadata.HashOriginal {
fileLogger.Info("skip already processed file (based on equal hash)", slog.String("input_hash", inputMetadata.Hash))
return
}
}
fileLogger.Info("start to process file")
reader, err := inputClient.GetReader(inputName)
if err != nil {
fileLogger.Error("fail to get reader for input file", slog.String("error", err.Error()))
return
}
writer, err := outputClient.GetWriter(outputName, inputMetadata)
if err != nil {
fileLogger.Error("fail to get writer for output file", slog.String("error", err.Error()))
return
}
if err := converter.Process(inputMetadata.ContentType, reader, writer); err != nil {
fileLogger.Error("fail to convert file", slog.String("error", err.Error()))
return
}
fileLogger.Info("successfully processed file", slog.String("input_hash", inputMetadata.Hash))
}(i, file)
}
wg.Wait()
generalLogger.Info("all files processed successfully, exiting")
}
|
