Improved single_file processor, refactoring

Reduce inlined image size, get page metadata before save and put into processing queue

adapters/processors/internal/mediainline.go

@@ -0,0 +1,255 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/disintegration/imaging"
+	"github.com/gabriel-vasile/mimetype"
+	"go.uber.org/zap"
+	"golang.org/x/net/html"
+)
+
+type MediaInline struct {
+	log    *zap.Logger
+	getter func(context.Context, string) (*http.Response, error)
+}
+
+func NewMediaInline(log *zap.Logger, getter func(context.Context, string) (*http.Response, error)) *MediaInline {
+	return &MediaInline{log: log, getter: getter}
+}
+
+func (m *MediaInline) Inline(ctx context.Context, reader io.Reader, pageURL string) (*html.Node, error) {
+	htmlNode, err := html.Parse(reader)
+	if err != nil {
+		return nil, fmt.Errorf("parse response body: %w", err)
+	}
+
+	baseURL, err := url.Parse(pageURL)
+	if err != nil {
+		return nil, fmt.Errorf("parse page url: %w", err)
+	}
+
+	m.visit(ctx, htmlNode, m.processorFunc, baseURL)
+
+	return htmlNode, nil
+}
+
+func (m *MediaInline) processorFunc(ctx context.Context, node *html.Node, baseURL *url.URL) error {
+	switch node.Data {
+	case "link":
+		if err := m.processHref(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process link %s: %w", node.Attr, err)
+		}
+
+	case "script", "img":
+		if err := m.processSrc(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process script %s: %w", node.Attr, err)
+		}
+
+	case "a":
+		if err := m.processAHref(node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process a href %s: %w", node.Attr, err)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processAHref(attrs []html.Attribute, baseURL *url.URL) error {
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "href":
+			attrs[idx].Val = normalizeURL(attr.Val, baseURL)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processHref(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "rel":
+			switch attr.Val {
+			case "stylesheet", "icon", "alternate icon", "shortcut icon", "manifest":
+				shouldProcess = true
+			}
+
+		case "href":
+			value = attr.Val
+			valueIdx = idx
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) processSrc(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "src":
+			value = attr.Val
+			valueIdx = idx
+			shouldProcess = true
+		case "data-src":
+			value = attr.Val
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) loadAndEncode(ctx context.Context, baseURL *url.URL, value string) (string, error) {
+	mime := "text/plain"
+
+	if value == "" {
+		return "", nil
+	}
+
+	normalizedURL := normalizeURL(value, baseURL)
+	if normalizedURL == "" {
+		return value, nil
+	}
+
+	response, err := m.getter(ctx, normalizedURL)
+	if err != nil {
+		m.log.Sugar().With(zap.Error(err)).Errorf("load %s", normalizedURL)
+		return value, nil
+	}
+
+	defer func() {
+		_ = response.Body.Close()
+	}()
+
+	cleanMime := func(s string) string {
+		s, _, _ = strings.Cut(s, "+")
+		return s
+	}
+
+	if ct := response.Header.Get("Content-Type"); ct != "" {
+		mime = ct
+	}
+
+	encodedVal, err := m.encodeResource(response.Body, &mime)
+	if err != nil {
+		return value, fmt.Errorf("encode resource: %w", err)
+	}
+
+	return fmt.Sprintf("data:%s;base64, %s", cleanMime(mime), encodedVal), nil
+}
+
+func (m *MediaInline) visit(ctx context.Context, n *html.Node, proc func(context.Context, *html.Node, *url.URL) error, baseURL *url.URL) {
+	if err := proc(ctx, n, baseURL); err != nil {
+		m.log.Error("process error", zap.Error(err))
+	}
+
+	if n.FirstChild != nil {
+		m.visit(ctx, n.FirstChild, proc, baseURL)
+	}
+
+	if n.NextSibling != nil {
+		m.visit(ctx, n.NextSibling, proc, baseURL)
+	}
+}
+
+func normalizeURL(resourceURL string, base *url.URL) string {
+	if strings.HasPrefix(resourceURL, "//") {
+		return "https:" + resourceURL
+	}
+
+	if strings.HasPrefix(resourceURL, "about:") {
+		return ""
+	}
+
+	parsedResourceURL, err := url.Parse(resourceURL)
+	if err != nil {
+		return resourceURL
+	}
+
+	reference := base.ResolveReference(parsedResourceURL)
+
+	return reference.String()
+}
+
+func (m *MediaInline) encodeResource(r io.Reader, mime *string) (string, error) {
+	all, err := io.ReadAll(r)
+	if err != nil {
+		return "", fmt.Errorf("read data: %w", err)
+	}
+
+	all, err = m.preprocessResource(all, mime)
+	if err != nil {
+		return "", fmt.Errorf("preprocess resource: %w", err)
+	}
+
+	return base64.StdEncoding.EncodeToString(all), nil
+}
+
+func (m *MediaInline) preprocessResource(data []byte, mime *string) ([]byte, error) {
+	detectedMime := mimetype.Detect(data)
+
+	switch {
+	case strings.HasPrefix(detectedMime.String(), "image"):
+		decodedImage, err := imaging.Decode(bytes.NewBuffer(data))
+		if err != nil {
+			m.log.Error("failed to decode image", zap.Error(err))
+
+			return data, nil
+		}
+
+		if size := decodedImage.Bounds().Size(); size.X > 1024 || size.Y > 1024 {
+			thumbnail := imaging.Thumbnail(decodedImage, 1024, 1024, imaging.Lanczos)
+			buf := bytes.NewBuffer(nil)
+
+			if err := imaging.Encode(buf, thumbnail, imaging.JPEG, imaging.JPEGQuality(90)); err != nil {
+				m.log.Error("failed to create resized image", zap.Error(err))
+
+				return data, nil
+			}
+
+			*mime = "image/jpeg"
+			m.log.Info("Resized")
+
+			return buf.Bytes(), nil
+		}
+	}
+
+	return data, nil
+}

adapters/processors/pdf.go

@@ -40,9 +40,9 @@ func (p *PDF) Process(_ context.Context, url string, cache *entity.Cache) ([]ent
 	opts := wkhtmltopdf.NewPageOptions()
 	opts.PrintMediaType.Set(p.cfg.MediaPrint)
 	opts.JavascriptDelay.Set(200)
-	opts.DisableJavascript.Set(true)
+	opts.DisableJavascript.Set(false)
 	opts.LoadErrorHandling.Set("ignore")
-	opts.LoadMediaErrorHandling.Set("ignore")
+	opts.LoadMediaErrorHandling.Set("skip")
 	opts.FooterRight.Set("[opts]")
 	opts.HeaderLeft.Set(url)
 	opts.HeaderRight.Set(time.Now().Format(time.DateOnly))
@@ -50,9 +50,9 @@ func (p *PDF) Process(_ context.Context, url string, cache *entity.Cache) ([]ent
 	opts.Zoom.Set(p.cfg.Zoom)
 	opts.ViewportSize.Set(p.cfg.Viewport)
 	opts.NoBackground.Set(true)
-	opts.DisableLocalFileAccess.Set(true)
-	opts.DisableExternalLinks.Set(true)
-	opts.DisableInternalLinks.Set(true)
+	opts.DisableLocalFileAccess.Set(false)
+	opts.DisableExternalLinks.Set(false)
+	opts.DisableInternalLinks.Set(false)
 
 	var page wkhtmltopdf.PageProvider
 	if len(cache.Get()) > 0 {

adapters/processors/processors.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/html"
 
 	"github.com/derfenix/webarchive/config"
@@ -22,7 +23,7 @@ type processor interface {
 	Process(ctx context.Context, url string, cache *entity.Cache) ([]entity.File, error)
 }
 
-func NewProcessors(cfg config.Config) (*Processors, error) {
+func NewProcessors(cfg config.Config, log *zap.Logger) (*Processors, error) {
 	jar, err := cookiejar.New(&cookiejar.Options{
 		PublicSuffixList: nil,
 	})
@@ -62,7 +63,7 @@ func NewProcessors(cfg config.Config) (*Processors, error) {
 		processors: map[entity.Format]processor{
 			entity.FormatHeaders:    NewHeaders(httpClient),
 			entity.FormatPDF:        NewPDF(cfg.PDF),
-			entity.FormatSingleFile: NewSingleFile(httpClient),
+			entity.FormatSingleFile: NewSingleFile(httpClient, log),
 		},
 	}
 

adapters/processors/processors_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
 
 	"github.com/derfenix/webarchive/config"
 	"github.com/derfenix/webarchive/entity"
@@ -18,7 +19,7 @@ func TestProcessors_GetMeta(t *testing.T) {
 	cfg, err := config.NewConfig(ctx)
 	require.NoError(t, err)
 
-	procs, err := NewProcessors(cfg)
+	procs, err := NewProcessors(cfg, zaptest.NewLogger(t))
 	require.NoError(t, err)
 
 	cache := entity.NewCache()

adapters/processors/singlefile.go

@@ -5,50 +5,46 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"net/url"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/html"
 
+	"github.com/derfenix/webarchive/adapters/processors/internal"
 	"github.com/derfenix/webarchive/entity"
 )
 
-func NewSingleFile(client *http.Client) *SingleFile {
-	return &SingleFile{client: client}
+func NewSingleFile(client *http.Client, log *zap.Logger) *SingleFile {
+	return &SingleFile{client: client, log: log}
 }
 
 type SingleFile struct {
 	client *http.Client
+	log    *zap.Logger
 }
 
-func (s *SingleFile) Process(ctx context.Context, url string, cache *entity.Cache) ([]entity.File, error) {
+func (s *SingleFile) Process(ctx context.Context, pageURL string, cache *entity.Cache) ([]entity.File, error) {
 	reader := cache.Reader()
 
 	if reader == nil {
-		response, err := s.get(ctx, url)
+		response, err := s.get(ctx, pageURL)
 		if err != nil {
 			return nil, err
 		}
 
-		if response.Body != nil {
-			defer func() {
-				_ = response.Body.Close()
-			}()
-		}
+		defer func() {
+			_ = response.Body.Close()
+		}()
 
 		reader = response.Body
 	}
 
-	htmlNode, err := html.Parse(reader)
+	inlinedHTML, err := internal.NewMediaInline(s.log, s.get).Inline(ctx, reader, pageURL)
 	if err != nil {
-		return nil, fmt.Errorf("parse response body: %w", err)
-	}
-
-	if err := s.process(ctx, htmlNode, url); err != nil {
-		return nil, fmt.Errorf("process: %w", err)
+		return nil, fmt.Errorf("inline media: %w", err)
 	}
 
 	buf := bytes.NewBuffer(nil)
-	if err := html.Render(buf, htmlNode); err != nil {
+	if err := html.Render(buf, inlinedHTML); err != nil {
 		return nil, fmt.Errorf("render result html: %w", err)
 	}
 
@@ -78,59 +74,3 @@ func (s *SingleFile) get(ctx context.Context, url string) (*http.Response, error
 
 	return response, nil
 }
-
-func (s *SingleFile) process(ctx context.Context, node *html.Node, pageURL string) error {
-	parsedURL, err := url.Parse(pageURL)
-	if err != nil {
-		return fmt.Errorf("parse page url: %w", err)
-	}
-
-	baseURL := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
-
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		var err error
-		switch child.Data {
-		case "head":
-			err = s.processHead(ctx, child, baseURL)
-
-		case "body":
-			err = s.processBody(ctx, child, baseURL)
-		}
-
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) processHead(ctx context.Context, node *html.Node, baseURL string) error {
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		switch child.Data {
-		case "link":
-			if err := s.processHref(ctx, child.Attr, baseURL); err != nil {
-				return fmt.Errorf("process link %s: %w", child.Attr, err)
-			}
-
-		case "script":
-			if err := s.processSrc(ctx, child.Attr, baseURL); err != nil {
-				return fmt.Errorf("process script %s: %w", child.Attr, err)
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) processBody(ctx context.Context, child *html.Node, url string) error {
-	return nil
-}
-
-func (s *SingleFile) processHref(ctx context.Context, attrs []html.Attribute, baseURL string) error {
-	return nil
-}
-
-func (s *SingleFile) processSrc(ctx context.Context, attrs []html.Attribute, baseURL string) error {
-	return nil
-}

adapters/repository/badger/page.go

@@ -192,7 +192,7 @@ func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
 				return fmt.Errorf("get item: %w", err)
 			}
 
-			if page.Status == entity.StatusNew {
+			if page.Status == entity.StatusNew || page.Status == entity.StatusProcessing {
 				//goland:noinspection GoVetCopyLock
 				pages = append(pages, page) //nolint:govet // didn't touch the lock here
 			}