Improved single_file processor, refactoring

Reduce inlined image size, get page metadata before save and put into processing queue
This commit is contained in:
2023-11-24 14:25:17 +03:00
parent 7e53519ca0
commit 870f13f7bf
14 changed files with 325 additions and 96 deletions

View File

@@ -66,6 +66,16 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
func (w *Worker) do(ctx context.Context, wg *sync.WaitGroup, page *Page, log *zap.Logger) {
defer wg.Done()
page.SetProcessing()
if err := w.pages.Save(ctx, page); err != nil {
w.log.Error(
"failed to save processing page",
zap.String("page_id", page.ID.String()),
zap.String("page_url", page.URL),
zap.Error(err),
)
}
page.Process(ctx, w.processor)
log.Debug("page processed")