fenix/webarchive
1
0
Fork 0
mirror of https://github.com/derfenix/webarchive.git synced 2026-03-11 22:40:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Improved single_file processor, refactoring

Browse Source 
Reduce inlined image size, get page metadata before save and put into processing queue
This commit is contained in:
derFenix
2023-11-24 14:25:17 +03:00
parent 7e53519ca0
commit 870f13f7bf
14 changed files with 325 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
entity/page.go
View File

@@ -3,6 +3,7 @@ package entity
import (
"context"
"fmt"
"runtime/debug"
"sync"
"time"
@@ -80,16 +81,18 @@ func (p *Page) SetProcessing() {
p.Status = StatusProcessing
}
func (p *Page) Process(ctx context.Context, processor Processor) {
innerWG := sync.WaitGroup{}
innerWG.Add(len(p.Formats))
func (p *Page) Prepare(ctx context.Context, processor Processor) {
meta, err := processor.GetMeta(ctx, p.URL, p.cache)
if err != nil {
p.Meta.Error = err.Error()
} else {
p.Meta = meta
}
}
func (p *Page) Process(ctx context.Context, processor Processor) {
innerWG := sync.WaitGroup{}
innerWG.Add(len(p.Formats))
results := Results{}
@@ -99,7 +102,7 @@ func (p *Page) Process(ctx context.Context, processor Processor) {
defer func() {
if err := recover(); err != nil {
results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v", err)})
results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v (%s)", err, string(debug.Stack()))})
}
}()