Bump golang.org/x/net from 0.37.0 to 0.38.0

Bumps [golang.org/x/net](https://github.com/golang/net) from 0.37.0 to 0.38.0.
- [Commits](https://github.com/golang/net/compare/v0.37.0...v0.38.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-version: 0.38.0
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

.deploystack/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "3"
+
+services:
+  webarchive:
+    image: ghcr.io/derfenix/webarchive:latest
+    environment:
+      LOGGING_DEBUG: "true"
+      API_ADDRESS: "0.0.0.0:5001"
+      PDF_DPI: "300"
+      DB_PATH: "/db"
+    volumes:
+      - ./db:/db
+    ports:
+      - "0.0.0.0:5001:5001"

.envrc

@@ -0,0 +1,5 @@
+export DIRENV_WARN_TIMEOUT=20s
+
+eval "$(devenv direnvrc)"
+
+use devenv

.github/workflows/release.yaml

@@ -1,9 +1,10 @@
+---
 name: release
 
-on:
+"on":
   push:
     tags:
-      - 'v*'
+      - "v*"
 
 jobs:
   release:
@@ -13,7 +14,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.x
+          go-version: 1.23.x
 
       - name: Checkout code
         uses: actions/checkout@v3

.github/workflows/test.yaml

@@ -1,6 +1,7 @@
+---
 name: test
 
-on:
+"on":
   pull_request:
   push:
     branches:
@@ -14,7 +15,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.20.x
+          go-version: 1.23.x
 
       - name: Checkout code
         uses: actions/checkout@v3

.gitignore

@@ -43,3 +43,15 @@ fabric.properties
 go.work
 test.http
 db
+http-client.env.json
+http-client.private.env.json
+
+# Devenv
+.devenv*
+devenv.local.nix
+
+# direnv
+.direnv
+
+# pre-commit
+.pre-commit-config.yaml

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

.idea/jsonSchemas.xml

@@ -3,11 +3,11 @@
   <component name="JsonSchemaMappingsProjectConfiguration">
     <state>
       <map>
-        <entry key="openapi">
+        <entry key="OpenAPI 3.0">
           <value>
             <SchemaInfo>
-              <option name="name" value="openapi" />
-              <option name="relativePathToSchema" value="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json" />
+              <option name="name" value="OpenAPI 3.0" />
+              <option name="relativePathToSchema" value="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/schemas/v3.0/schema.json" />
               <option name="applicationDefined" value="true" />
               <option name="patterns">
                 <list>

.idea/swagger-settings.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SwaggerSettings">
+    <option name="defaultPreviewType" value="SWAGGER_UI" />
+  </component>
+</project>

.idea/yamllint.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="YamllintSettings">
+    <enabled>true</enabled>
+    <binPath>/usr/bin/yamllint</binPath>
+  </component>
+</project>

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as builder
+FROM golang:1.23-alpine AS builder
 
 WORKDIR /project
 ADD go.* ./

README.md

@@ -41,6 +41,16 @@ variables:
 *Note*: Prefix **WEBARCHIVE_** can be used with the environment variable names 
 in case of any conflicts.
 
+## ⚡ One-Click Deploy
+
+| Cloud Provider | Deploy Button |
+|----------------|---------------|
+| AWS | <a href="https://deploystack.io/deploy/derfenix-webarchive?provider=aws&language=cfn"><img src="https://raw.githubusercontent.com/deploystackio/deploy-templates/refs/heads/main/.assets/img/aws.svg" height="38"></a> |
+| DigitalOcean | <a href="https://deploystack.io/deploy/derfenix-webarchive?provider=do&language=dop"><img src="https://raw.githubusercontent.com/deploystackio/deploy-templates/refs/heads/main/.assets/img/do.svg" height="38"></a> |
+| Render | <a href="https://deploystack.io/deploy/derfenix-webarchive?provider=rnd&language=rnd"><img src="https://raw.githubusercontent.com/deploystackio/deploy-templates/refs/heads/main/.assets/img/rnd.svg" height="38"></a> |
+
+<sub>Generated by <a href="https://deploystack.io/c/derfenix-webarchive" target="_blank">DeployStack.io</a></sub>
+
 ## Usage
 
 ### 1. Start the server
@@ -116,3 +126,4 @@ curl -X GET --location "http://localhost:5001/api/v1/pages" | jq .
 - [ ] Multi-user access
 - [ ] Support SQL database with or without separate files storage
 - [ ] Tags/Categories
+- [ ] Save page to markdown

adapters/processors/headers.go

@@ -17,7 +17,7 @@ type Headers struct {
 	client *http.Client
 }
 
-func (h *Headers) Process(ctx context.Context, url string) ([]entity.File, error) {
+func (h *Headers) Process(ctx context.Context, url string, _ *entity.Cache) ([]entity.File, error) {
 	var (
 		headersFile entity.File
 		err         error

adapters/processors/internal/mediainline.go

@@ -0,0 +1,255 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/disintegration/imaging"
+	"github.com/gabriel-vasile/mimetype"
+	"go.uber.org/zap"
+	"golang.org/x/net/html"
+)
+
+type MediaInline struct {
+	log    *zap.Logger
+	getter func(context.Context, string) (*http.Response, error)
+}
+
+func NewMediaInline(log *zap.Logger, getter func(context.Context, string) (*http.Response, error)) *MediaInline {
+	return &MediaInline{log: log, getter: getter}
+}
+
+func (m *MediaInline) Inline(ctx context.Context, reader io.Reader, pageURL string) (*html.Node, error) {
+	htmlNode, err := html.Parse(reader)
+	if err != nil {
+		return nil, fmt.Errorf("parse response body: %w", err)
+	}
+
+	baseURL, err := url.Parse(pageURL)
+	if err != nil {
+		return nil, fmt.Errorf("parse page url: %w", err)
+	}
+
+	m.visit(ctx, htmlNode, m.processorFunc, baseURL)
+
+	return htmlNode, nil
+}
+
+func (m *MediaInline) processorFunc(ctx context.Context, node *html.Node, baseURL *url.URL) error {
+	switch node.Data {
+	case "link":
+		if err := m.processHref(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process link %s: %w", node.Attr, err)
+		}
+
+	case "script", "img":
+		if err := m.processSrc(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process script %s: %w", node.Attr, err)
+		}
+
+	case "a":
+		if err := m.processAHref(node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process a href %s: %w", node.Attr, err)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processAHref(attrs []html.Attribute, baseURL *url.URL) error {
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "href":
+			attrs[idx].Val = normalizeURL(attr.Val, baseURL)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processHref(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "rel":
+			switch attr.Val {
+			case "stylesheet", "icon", "alternate icon", "shortcut icon", "manifest":
+				shouldProcess = true
+			}
+
+		case "href":
+			value = attr.Val
+			valueIdx = idx
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) processSrc(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "src":
+			value = attr.Val
+			valueIdx = idx
+			shouldProcess = true
+		case "data-src":
+			value = attr.Val
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) loadAndEncode(ctx context.Context, baseURL *url.URL, value string) (string, error) {
+	mime := "text/plain"
+
+	if value == "" {
+		return "", nil
+	}
+
+	normalizedURL := normalizeURL(value, baseURL)
+	if normalizedURL == "" {
+		return value, nil
+	}
+
+	response, err := m.getter(ctx, normalizedURL)
+	if err != nil {
+		m.log.Sugar().With(zap.Error(err)).Errorf("load %s", normalizedURL)
+		return value, nil
+	}
+
+	defer func() {
+		_ = response.Body.Close()
+	}()
+
+	cleanMime := func(s string) string {
+		s, _, _ = strings.Cut(s, "+")
+		return s
+	}
+
+	if ct := response.Header.Get("Content-Type"); ct != "" {
+		mime = ct
+	}
+
+	encodedVal, err := m.encodeResource(response.Body, &mime)
+	if err != nil {
+		return value, fmt.Errorf("encode resource: %w", err)
+	}
+
+	return fmt.Sprintf("data:%s;base64, %s", cleanMime(mime), encodedVal), nil
+}
+
+func (m *MediaInline) visit(ctx context.Context, n *html.Node, proc func(context.Context, *html.Node, *url.URL) error, baseURL *url.URL) {
+	if err := proc(ctx, n, baseURL); err != nil {
+		m.log.Error("process error", zap.Error(err))
+	}
+
+	if n.FirstChild != nil {
+		m.visit(ctx, n.FirstChild, proc, baseURL)
+	}
+
+	if n.NextSibling != nil {
+		m.visit(ctx, n.NextSibling, proc, baseURL)
+	}
+}
+
+func normalizeURL(resourceURL string, base *url.URL) string {
+	if strings.HasPrefix(resourceURL, "//") {
+		return "https:" + resourceURL
+	}
+
+	if strings.HasPrefix(resourceURL, "about:") {
+		return ""
+	}
+
+	parsedResourceURL, err := url.Parse(resourceURL)
+	if err != nil {
+		return resourceURL
+	}
+
+	reference := base.ResolveReference(parsedResourceURL)
+
+	return reference.String()
+}
+
+func (m *MediaInline) encodeResource(r io.Reader, mime *string) (string, error) {
+	all, err := io.ReadAll(r)
+	if err != nil {
+		return "", fmt.Errorf("read data: %w", err)
+	}
+
+	all, err = m.preprocessResource(all, mime)
+	if err != nil {
+		return "", fmt.Errorf("preprocess resource: %w", err)
+	}
+
+	return base64.StdEncoding.EncodeToString(all), nil
+}
+
+func (m *MediaInline) preprocessResource(data []byte, mime *string) ([]byte, error) {
+	detectedMime := mimetype.Detect(data)
+
+	switch {
+	case strings.HasPrefix(detectedMime.String(), "image"):
+		decodedImage, err := imaging.Decode(bytes.NewBuffer(data))
+		if err != nil {
+			m.log.Error("failed to decode image", zap.Error(err))
+
+			return data, nil
+		}
+
+		if size := decodedImage.Bounds().Size(); size.X > 1024 || size.Y > 1024 {
+			thumbnail := imaging.Thumbnail(decodedImage, 1024, 1024, imaging.Lanczos)
+			buf := bytes.NewBuffer(nil)
+
+			if err := imaging.Encode(buf, thumbnail, imaging.JPEG, imaging.JPEGQuality(90)); err != nil {
+				m.log.Error("failed to create resized image", zap.Error(err))
+
+				return data, nil
+			}
+
+			*mime = "image/jpeg"
+			m.log.Info("Resized")
+
+			return buf.Bytes(), nil
+		}
+	}
+
+	return data, nil
+}

adapters/processors/pdf.go

@@ -19,7 +19,7 @@ type PDF struct {
 	cfg config.PDF
 }
 
-func (p *PDF) Process(_ context.Context, url string) ([]entity.File, error) {
+func (p *PDF) Process(_ context.Context, url string, cache *entity.Cache) ([]entity.File, error) {
 	gen, err := wkhtmltopdf.NewPDFGenerator()
 	if err != nil {
 		return nil, fmt.Errorf("new pdf generator: %w", err)
@@ -37,17 +37,29 @@ func (p *PDF) Process(_ context.Context, url string) ([]entity.File, error) {
 	gen.Grayscale.Set(p.cfg.Grayscale)
 	gen.Title.Set(url)
 
-	page := wkhtmltopdf.NewPage(url)
-	page.PrintMediaType.Set(p.cfg.MediaPrint)
-	page.JavascriptDelay.Set(200)
-	page.LoadMediaErrorHandling.Set("ignore")
-	page.FooterRight.Set("[page]")
-	page.HeaderLeft.Set(url)
-	page.HeaderRight.Set(time.Now().Format(time.DateOnly))
-	page.FooterFontSize.Set(10)
-	page.Zoom.Set(p.cfg.Zoom)
-	page.ViewportSize.Set(p.cfg.Viewport)
-	page.NoBackground.Set(true)
+	opts := wkhtmltopdf.NewPageOptions()
+	opts.PrintMediaType.Set(p.cfg.MediaPrint)
+	opts.JavascriptDelay.Set(200)
+	opts.DisableJavascript.Set(false)
+	opts.LoadErrorHandling.Set("ignore")
+	opts.LoadMediaErrorHandling.Set("skip")
+	opts.FooterRight.Set("[opts]")
+	opts.HeaderLeft.Set(url)
+	opts.HeaderRight.Set(time.Now().Format(time.DateOnly))
+	opts.FooterFontSize.Set(10)
+	opts.Zoom.Set(p.cfg.Zoom)
+	opts.ViewportSize.Set(p.cfg.Viewport)
+	opts.NoBackground.Set(true)
+	opts.DisableLocalFileAccess.Set(false)
+	opts.DisableExternalLinks.Set(false)
+	opts.DisableInternalLinks.Set(false)
+
+	var page wkhtmltopdf.PageProvider
+	if len(cache.Get()) > 0 {
+		page = &wkhtmltopdf.PageReader{Input: cache.Reader(), PageOptions: opts}
+	} else {
+		page = &wkhtmltopdf.Page{Input: url, PageOptions: opts}
+	}
 
 	gen.AddPage(page)
 

adapters/processors/processors.go

@@ -3,22 +3,27 @@ package processors
 import (
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
+	"strings"
 	"time"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/html"
 
 	"github.com/derfenix/webarchive/config"
 	"github.com/derfenix/webarchive/entity"
 )
 
+const defaultEncoding = "utf-8"
+
 type processor interface {
-	Process(ctx context.Context, url string) ([]entity.File, error)
+	Process(ctx context.Context, url string, cache *entity.Cache) ([]entity.File, error)
 }
 
-func NewProcessors(cfg config.Config) (*Processors, error) {
+func NewProcessors(cfg config.Config, log *zap.Logger) (*Processors, error) {
 	jar, err := cookiejar.New(&cookiejar.Options{
 		PublicSuffixList: nil,
 	})
@@ -58,7 +63,7 @@ func NewProcessors(cfg config.Config) (*Processors, error) {
 		processors: map[entity.Format]processor{
 			entity.FormatHeaders:    NewHeaders(httpClient),
 			entity.FormatPDF:        NewPDF(cfg.PDF),
-			entity.FormatSingleFile: NewSingleFile(httpClient),
+			entity.FormatSingleFile: NewSingleFile(httpClient, log),
 		},
 	}
 
@@ -70,7 +75,7 @@ type Processors struct {
 	client     *http.Client
 }
 
-func (p *Processors) Process(ctx context.Context, format entity.Format, url string) entity.Result {
+func (p *Processors) Process(ctx context.Context, format entity.Format, url string, cache *entity.Cache) entity.Result {
 	result := entity.Result{Format: format}
 
 	proc, ok := p.processors[format]
@@ -80,7 +85,7 @@ func (p *Processors) Process(ctx context.Context, format entity.Format, url stri
 		return result
 	}
 
-	files, err := proc.Process(ctx, url)
+	files, err := proc.Process(ctx, url, cache)
 	if err != nil {
 		result.Err = fmt.Errorf("process: %w", err)
 
@@ -98,7 +103,7 @@ func (p *Processors) OverrideProcessor(format entity.Format, proc processor) err
 	return nil
 }
 
-func (p *Processors) GetMeta(ctx context.Context, url string) (entity.Meta, error) {
+func (p *Processors) GetMeta(ctx context.Context, url string, cache *entity.Cache) (entity.Meta, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return entity.Meta{}, fmt.Errorf("new request: %w", err)
@@ -121,13 +126,37 @@ func (p *Processors) GetMeta(ctx context.Context, url string) (entity.Meta, erro
 		_ = response.Body.Close()
 	}()
 
-	htmlNode, err := html.Parse(response.Body)
+	tee := io.TeeReader(response.Body, cache)
+
+	htmlNode, err := html.Parse(tee)
 	if err != nil {
 		return entity.Meta{}, fmt.Errorf("parse response body: %w", err)
 	}
 
+	var fc *html.Node
+	for fc = htmlNode.FirstChild; fc != nil && fc.Data != "html"; fc = fc.NextSibling {
+	}
+
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
+	fc = fc.NextSibling
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
+	for fc = fc.FirstChild; fc != nil && fc.Data != "head"; fc = fc.NextSibling {
+		fmt.Println(fc.Data)
+	}
+
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
 	meta := entity.Meta{}
-	getMetaData(htmlNode, &meta)
+	getMetaData(fc, &meta)
+	meta.Encoding = encodingFromHeader(response.Header)
 
 	return meta, nil
 }
@@ -156,3 +185,19 @@ func getMetaData(n *html.Node, meta *entity.Meta) {
 		getMetaData(c, meta)
 	}
 }
+
+func encodingFromHeader(headers http.Header) string {
+	var foundEncoding bool
+	var encoding string
+
+	_, encoding, foundEncoding = strings.Cut(headers.Get("Content-Type"), "; ")
+	if foundEncoding {
+		_, encoding, foundEncoding = strings.Cut(encoding, "=")
+	}
+
+	if !foundEncoding {
+		encoding = defaultEncoding
+	}
+
+	return encoding
+}

adapters/processors/processors_test.go

@@ -0,0 +1,30 @@
+package processors
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+
+	"github.com/derfenix/webarchive/config"
+	"github.com/derfenix/webarchive/entity"
+)
+
+func TestProcessors_GetMeta(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	cfg, err := config.NewConfig(ctx)
+	require.NoError(t, err)
+
+	procs, err := NewProcessors(cfg, zaptest.NewLogger(t))
+	require.NoError(t, err)
+
+	cache := entity.NewCache()
+
+	meta, err := procs.GetMeta(ctx, "https://habr.com/ru/companies/wirenboard/articles/722718/", cache)
+	require.NoError(t, err)
+	assert.Equal(t, "Сколько стоит умный дом? Рассказываю, как строил свой и что получилось за 1000 руб./м² / Хабр", meta.Title)
+}

adapters/processors/singlefile.go

@@ -3,30 +3,57 @@ package processors
 import (
 	"bytes"
 	"context"
-	"encoding/base64"
 	"fmt"
-	"io"
 	"net/http"
-	"net/url"
-	"strings"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/html"
-	"golang.org/x/net/html/atom"
 
+	"github.com/derfenix/webarchive/adapters/processors/internal"
 	"github.com/derfenix/webarchive/entity"
 )
 
-const defaultEncoding = "utf-8"
-
-func NewSingleFile(client *http.Client) *SingleFile {
-	return &SingleFile{client: client}
+func NewSingleFile(client *http.Client, log *zap.Logger) *SingleFile {
+	return &SingleFile{client: client, log: log}
 }
 
 type SingleFile struct {
 	client *http.Client
+	log    *zap.Logger
 }
 
-func (s *SingleFile) Process(ctx context.Context, url string) ([]entity.File, error) {
+func (s *SingleFile) Process(ctx context.Context, pageURL string, cache *entity.Cache) ([]entity.File, error) {
+	reader := cache.Reader()
+
+	if reader == nil {
+		response, err := s.get(ctx, pageURL)
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			_ = response.Body.Close()
+		}()
+
+		reader = response.Body
+	}
+
+	inlinedHTML, err := internal.NewMediaInline(s.log, s.get).Inline(ctx, reader, pageURL)
+	if err != nil {
+		return nil, fmt.Errorf("inline media: %w", err)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	if err := html.Render(buf, inlinedHTML); err != nil {
+		return nil, fmt.Errorf("render result html: %w", err)
+	}
+
+	htmlFile := entity.NewFile("page.html", buf.Bytes())
+
+	return []entity.File{htmlFile}, nil
+}
+
+func (s *SingleFile) get(ctx context.Context, url string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("new request: %w", err)
@@ -45,180 +72,5 @@ func (s *SingleFile) Process(ctx context.Context, url string) ([]entity.File, er
 		return nil, fmt.Errorf("empty response body")
 	}
 
-	defer func() {
-		_ = response.Body.Close()
-	}()
-
-	htmlNode, err := html.Parse(response.Body)
-	if err != nil {
-		return nil, fmt.Errorf("parse response body: %w", err)
-	}
-
-	if err := s.crawl(ctx, htmlNode, baseURL(url), getEncoding(response)); err != nil {
-		return nil, fmt.Errorf("crawl: %w", err)
-	}
-
-	buf := bytes.NewBuffer(nil)
-	if err := html.Render(buf, htmlNode); err != nil {
-		return nil, fmt.Errorf("render result html: %w", err)
-	}
-
-	htmlFile := entity.NewFile("page.html", buf.Bytes())
-
-	return []entity.File{htmlFile}, nil
-}
-
-func (s *SingleFile) crawl(ctx context.Context, node *html.Node, baseURL string, encoding string) error {
-	if node.Data == "head" {
-		s.setCharset(node, encoding)
-	}
-
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		if child.Type == html.ElementNode {
-			if err := s.findAndReplaceResources(ctx, child, baseURL); err != nil {
-				return err
-			}
-		}
-
-		if err := s.crawl(ctx, child, baseURL, encoding); err != nil {
-			return fmt.Errorf("crawl child %s: %w", child.Data, err)
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) findAndReplaceResources(ctx context.Context, node *html.Node, baseURL string) error {
-	switch node.DataAtom {
-	case atom.Img, atom.Image, atom.Script, atom.Style:
-		err := s.replaceResource(ctx, node, baseURL)
-		if err != nil {
-			return err
-		}
-
-	case atom.Link:
-		for _, attribute := range node.Attr {
-			if attribute.Key == "rel" && (attribute.Val == "stylesheet") {
-				if err := s.replaceResource(ctx, node, baseURL); err != nil {
-					return err
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) replaceResource(ctx context.Context, node *html.Node, baseURL string) error {
-	for i, attribute := range node.Attr {
-		if attribute.Key == "src" || attribute.Key == "href" {
-			encoded, contentType, err := s.loadResource(ctx, attribute.Val, baseURL)
-			if err != nil {
-				return fmt.Errorf("load resource for %s: %w", node.Data, err)
-			}
-
-			if len(encoded) == 0 {
-				attribute.Val = ""
-
-			} else {
-				attribute.Val = fmt.Sprintf("data:%s;base64, %s", contentType, encoded)
-			}
-
-			node.Attr[i] = attribute
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) loadResource(ctx context.Context, val, baseURL string) ([]byte, string, error) {
-	if !strings.HasPrefix(val, "http://") && !strings.HasPrefix(val, "https://") {
-		var err error
-		val, err = url.JoinPath(baseURL, val)
-		if err != nil {
-			return nil, "", fmt.Errorf("join base path %s and url %s: %w", baseURL, val, err)
-		}
-		val, err = url.PathUnescape(val)
-		if err != nil {
-			return nil, "", fmt.Errorf("unescape path %s: %w", val, err)
-		}
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, val, nil)
-	if err != nil {
-		return nil, "", fmt.Errorf("new request: %w", err)
-	}
-
-	response, err := s.client.Do(req)
-	if err != nil {
-		return nil, "", fmt.Errorf("do request: %w", err)
-	}
-
-	defer func() {
-		if response.Body != nil {
-			_ = response.Body.Close()
-		}
-	}()
-
-	if response.StatusCode != http.StatusOK {
-		return []byte{}, "", nil
-	}
-
-	raw, err := io.ReadAll(response.Body)
-	if err != nil {
-		return nil, "", fmt.Errorf("read body: %w", err)
-	}
-
-	encoded := make([]byte, base64.StdEncoding.EncodedLen(len(raw)))
-	base64.StdEncoding.Encode(encoded, raw)
-
-	return encoded, response.Header.Get("Content-Type"), nil
-}
-
-func (s *SingleFile) setCharset(node *html.Node, encoding string) {
-	var charsetExists bool
-
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		if child.Data == "meta" {
-			for _, attribute := range child.Attr {
-				if attribute.Key == "charset" {
-					charsetExists = true
-				}
-			}
-		}
-	}
-
-	if !charsetExists {
-		node.AppendChild(&html.Node{
-			Type:     html.ElementNode,
-			DataAtom: atom.Meta,
-			Data:     "meta",
-			Attr: []html.Attribute{
-				{
-					Key: "charset",
-					Val: encoding,
-				},
-			},
-		})
-	}
-}
-
-func baseURL(val string) string {
-	parsed, err := url.Parse(val)
-	if err != nil {
-		return val
-	}
-
-	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
-}
-
-func getEncoding(response *http.Response) string {
-	_, encoding, found := strings.Cut(response.Header.Get("Content-Type"), "charset=")
-	if !found {
-		return defaultEncoding
-	}
-
-	encoding = strings.TrimSpace(encoding)
-
-	return encoding
+	return response, nil
 }

adapters/repository/badger.go

@@ -1,4 +1,4 @@
-package badger
+package repository
 
 import (
 	"errors"

adapters/repository/badger/page.go

@@ -8,6 +8,8 @@ import (
 	"github.com/dgraph-io/badger/v4"
 	"github.com/google/uuid"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/entity"
 )
 
@@ -24,7 +26,9 @@ type Page struct {
 }
 
 func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
-	page := entity.Page{ID: pageID}
+	page := entity.Page{}
+	page.ID = pageID
+
 	var file *entity.File
 
 	err := p.db.View(func(txn *badger.Txn) error {
@@ -44,9 +48,9 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil
 			return fmt.Errorf("get value: %w", err)
 		}
 
-		for i := range page.Results.Results() {
-			for j := range page.Results.Results()[i].Files {
-				ff := &page.Results.Results()[i].Files[j]
+		for i := range page.Results {
+			for j := range page.Results[i].Files {
+				ff := &page.Results[i].Files[j]
 
 				if ff.ID == fileID {
 					file = ff
@@ -66,7 +70,7 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil
 
 func (p *Page) Save(_ context.Context, page *entity.Page) error {
 	if p.db.IsClosed() {
-		return ErrDBClosed
+		return repository.ErrDBClosed
 	}
 
 	marshaled, err := marshal(page)
@@ -88,16 +92,17 @@ func (p *Page) Save(_ context.Context, page *entity.Page) error {
 }
 
 func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
-	site := entity.Page{ID: id}
+	page := entity.Page{}
+	page.ID = id
 
 	err := p.db.View(func(txn *badger.Txn) error {
-		data, err := txn.Get(p.key(&site))
+		data, err := txn.Get(p.key(&page))
 		if err != nil {
 			return fmt.Errorf("get data: %w", err)
 		}
 
 		err = data.Value(func(val []byte) error {
-			if err := unmarshal(val, &site); err != nil {
+			if err := unmarshal(val, &page); err != nil {
 				return fmt.Errorf("unmarshal data: %w", err)
 			}
 
@@ -113,7 +118,7 @@ func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
 		return nil, fmt.Errorf("view: %w", err)
 	}
 
-	return &site, nil
+	return &page, nil
 }
 
 func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
@@ -143,16 +148,7 @@ func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
 				return fmt.Errorf("get item: %w", err)
 			}
 
-			pages = append(pages, &entity.Page{
-				ID:          page.ID,
-				URL:         page.URL,
-				Description: page.Description,
-				Created:     page.Created,
-				Formats:     page.Formats,
-				Version:     page.Version,
-				Status:      page.Status,
-				Meta:        page.Meta,
-			})
+			pages = append(pages, &page)
 		}
 
 		return nil
@@ -169,8 +165,8 @@ func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
 	return pages, nil
 }
 
-func (p *Page) ListUnprocessed(ctx context.Context) ([]*entity.Page, error) {
-	pages := make([]*entity.Page, 0, 100)
+func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
+	pages := make([]entity.Page, 0, 100)
 
 	err := p.db.View(func(txn *badger.Txn) error {
 		iterator := txn.NewIterator(badger.DefaultIteratorOptions)
@@ -196,20 +192,11 @@ func (p *Page) ListUnprocessed(ctx context.Context) ([]*entity.Page, error) {
 				return fmt.Errorf("get item: %w", err)
 			}
 
-			if page.Status != entity.StatusProcessing {
-				continue
+			if page.Status == entity.StatusNew || page.Status == entity.StatusProcessing {
+				//goland:noinspection GoVetCopyLock
+				pages = append(pages, page) //nolint:govet // didn't touch the lock here
 			}
 
-			pages = append(pages, &entity.Page{
-				ID:          page.ID,
-				URL:         page.URL,
-				Description: page.Description,
-				Created:     page.Created,
-				Formats:     page.Formats,
-				Version:     page.Version,
-				Status:      page.Status,
-				Meta:        page.Meta,
-			})
 		}
 
 		return nil

adapters/repository/badger/page_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zaptest"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/entity"
 )
 
@@ -31,7 +33,7 @@ func TestSite(t *testing.T) {
 
 	log := zaptest.NewLogger(t)
 
-	db, err := NewBadger(tempDir, log.Named("db"))
+	db, err := repository.NewBadger(tempDir, log.Named("db"))
 	require.NoError(t, err)
 
 	siteRepo, err := NewPage(db)
@@ -49,12 +51,16 @@ func TestSite(t *testing.T) {
 		storedSite, err := siteRepo.Get(ctx, site.ID)
 		require.NoError(t, err)
 
-		assert.Equal(t, site, storedSite)
+		assert.Equal(t, site.ID, storedSite.ID)
+		assert.Equal(t, site.URL, storedSite.URL)
+		assert.Equal(t, site.Status, storedSite.Status)
 
 		all, err := siteRepo.ListAll(ctx)
 		require.NoError(t, err)
 		require.Len(t, all, 1)
 
-		assert.Equal(t, site, all[0])
+		assert.Equal(t, site.ID, all[0].ID)
+		assert.Equal(t, site.URL, all[0].URL)
+		assert.Equal(t, site.Status, all[0].Status)
 	})
 }

adapters/repository/badgers3/marshal.go

@@ -0,0 +1,13 @@
+package badgers3
+
+import (
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+func marshal(v interface{}) ([]byte, error) {
+	return msgpack.Marshal(v)
+}
+
+func unmarshal(b []byte, v interface{}) error { //nolint:unused // will use later
+	return msgpack.Unmarshal(b, v)
+}

adapters/repository/badgers3/page.go

@@ -0,0 +1,119 @@
+package badgers3
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/derfenix/webarchive/adapters/repository"
+	"github.com/derfenix/webarchive/entity"
+	"github.com/dgraph-io/badger/v4"
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+)
+
+func NewPage(db *badger.DB, s3 *minio.Client, bucketName string) (*Page, error) {
+	return &Page{
+		db:         db,
+		s3:         s3,
+		prefix:     []byte("pages3:"),
+		bucketName: bucketName,
+	}, nil
+}
+
+type Page struct {
+	db         *badger.DB
+	s3         *minio.Client
+	prefix     []byte
+	bucketName string
+}
+
+func (p *Page) ListAll(ctx context.Context) ([]*entity.PageBase, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) Get(ctx context.Context, id uuid.UUID) (*entity.Page, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) GetFile(ctx context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) Save(ctx context.Context, page *entity.Page) error {
+	if p.db.IsClosed() {
+		return repository.ErrDBClosed
+	}
+
+	marshaled, err := marshal(page.PageBase)
+	if err != nil {
+		return fmt.Errorf("marshal data: %w", err)
+	}
+
+	if err := p.db.Update(func(txn *badger.Txn) error {
+		if err := txn.Set(p.key(page), marshaled); err != nil {
+			return fmt.Errorf("put data: %w", err)
+		}
+
+		return nil
+	}); err != nil {
+		return fmt.Errorf("update db: %w", err)
+	}
+
+	snowball := make(chan minio.SnowballObject, 1)
+
+	go func() {
+		defer close(snowball)
+
+		for _, result := range page.Results {
+			for _, file := range result.Files {
+				for {
+					if ctx.Err() != nil {
+						return
+					}
+
+					if len(snowball) < cap(snowball) {
+						break
+					}
+				}
+
+				snowball <- minio.SnowballObject{
+					Key:     file.ID.String(),
+					Size:    int64(len(file.Data)),
+					ModTime: time.Now(),
+					Content: bytes.NewReader(file.Data),
+				}
+			}
+		}
+	}()
+
+	if err = p.s3.PutObjectsSnowball(ctx, p.bucketName, minio.SnowballOptions{Compress: true}, snowball); err != nil {
+		if dErr := p.db.Update(func(txn *badger.Txn) error {
+			if err := txn.Delete(p.key(page)); err != nil {
+				return fmt.Errorf("put data: %w", err)
+			}
+
+			return nil
+		}); dErr != nil {
+			err = errors.Join(err, dErr)
+		}
+
+		return fmt.Errorf("store files to s3: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) key(site *entity.Page) []byte {
+	return append(p.prefix, []byte(site.ID.String())...)
+}

api/gen.go

@@ -1,3 +1,3 @@
 package api
 
-//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.60.1 --target ./openapi -package openapi --clean openapi.yaml
+//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.77.0 --target ./openapi -package openapi --clean openapi.yaml

api/openapi.yaml

@@ -1,4 +1,5 @@
-openapi: 3.1.0
+---
+openapi: 3.0.3
 info:
   title: Sample API
   description: API description in Markdown.
@@ -125,7 +126,7 @@ paths:
         200:
           description: File content
           content:
-            application/pdf: { }
+            application/pdf: {}
             text/plain:
               schema:
                 type: string

api/openapi/oas_cfg_gen.go

@@ -7,7 +7,6 @@ import (
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/trace"
 
 	ht "github.com/ogen-go/ogen/http"
@@ -40,7 +39,7 @@ func (cfg *otelConfig) initOTEL() {
 		cfg.TracerProvider = otel.GetTracerProvider()
 	}
 	if cfg.MeterProvider == nil {
-		cfg.MeterProvider = metric.NewNoopMeterProvider()
+		cfg.MeterProvider = otel.GetMeterProvider()
 	}
 	cfg.Tracer = cfg.TracerProvider.Tracer(otelogen.Name,
 		trace.WithInstrumentationVersion(otelogen.SemVersion()),
@@ -99,9 +98,9 @@ func newServerConfig(opts ...ServerOption) serverConfig {
 
 type baseServer struct {
 	cfg      serverConfig
-	requests instrument.Int64Counter
-	errors   instrument.Int64Counter
-	duration instrument.Int64Histogram
+	requests metric.Int64Counter
+	errors   metric.Int64Counter
+	duration metric.Float64Histogram
 }
 
 func (s baseServer) notFound(w http.ResponseWriter, r *http.Request) {
@@ -120,7 +119,7 @@ func (cfg serverConfig) baseServer() (s baseServer, err error) {
 	if s.errors, err = s.cfg.Meter.Int64Counter(otelogen.ServerErrorsCount); err != nil {
 		return s, err
 	}
-	if s.duration, err = s.cfg.Meter.Int64Histogram(otelogen.ServerDuration); err != nil {
+	if s.duration, err = s.cfg.Meter.Float64Histogram(otelogen.ServerDuration); err != nil {
 		return s, err
 	}
 	return s, nil
@@ -162,9 +161,9 @@ func newClientConfig(opts ...ClientOption) clientConfig {
 
 type baseClient struct {
 	cfg      clientConfig
-	requests instrument.Int64Counter
-	errors   instrument.Int64Counter
-	duration instrument.Int64Histogram
+	requests metric.Int64Counter
+	errors   metric.Int64Counter
+	duration metric.Float64Histogram
 }
 
 func (cfg clientConfig) baseClient() (c baseClient, err error) {
@@ -175,7 +174,7 @@ func (cfg clientConfig) baseClient() (c baseClient, err error) {
 	if c.errors, err = c.cfg.Meter.Int64Counter(otelogen.ClientErrorsCount); err != nil {
 		return c, err
 	}
-	if c.duration, err = c.cfg.Meter.Int64Histogram(otelogen.ClientDuration); err != nil {
+	if c.duration, err = c.cfg.Meter.Float64Histogram(otelogen.ClientDuration); err != nil {
 		return c, err
 	}
 	return c, nil
@@ -200,7 +199,7 @@ func WithTracerProvider(provider trace.TracerProvider) Option {
 
 // WithMeterProvider specifies a meter provider to use for creating a meter.
 //
-// If none is specified, the metric.NewNoopMeterProvider is used.
+// If none is specified, the otel.GetMeterProvider() is used.
 func WithMeterProvider(provider metric.MeterProvider) Option {
 	return otelOptionFunc(func(cfg *otelConfig) {
 		if provider != nil {

api/openapi/oas_client_gen.go

@@ -11,6 +11,8 @@ import (
 	"github.com/go-faster/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/ogen-go/ogen/conv"
@@ -19,6 +21,34 @@ import (
 	"github.com/ogen-go/ogen/uri"
 )
 
+// Invoker invokes operations described by OpenAPI v3 specification.
+type Invoker interface {
+	// AddPage invokes addPage operation.
+	//
+	// Add new page.
+	//
+	// POST /pages
+	AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error)
+	// GetFile invokes getFile operation.
+	//
+	// Get file content.
+	//
+	// GET /pages/{id}/file/{file_id}
+	GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error)
+	// GetPage invokes getPage operation.
+	//
+	// Get page details.
+	//
+	// GET /pages/{id}
+	GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error)
+	// GetPages invokes getPages operation.
+	//
+	// Get all pages.
+	//
+	// GET /pages
+	GetPages(ctx context.Context) (Pages, error)
+}
+
 // Client implements OAS client.
 type Client struct {
 	serverURL *url.URL
@@ -78,19 +108,20 @@ func (c *Client) requestURL(ctx context.Context) *url.URL {
 // POST /pages
 func (c *Client) AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error) {
 	res, err := c.sendAddPage(ctx, request, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (res AddPageRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("addPage"),
+		semconv.HTTPMethodKey.String("POST"),
+		semconv.HTTPRouteKey.String("/pages"),
 	}
 	// Validate request before sending.
 	if err := func() error {
-		if request.Set {
+		if value, ok := request.Get(); ok {
 			if err := func() error {
-				if err := request.Value.Validate(); err != nil {
+				if err := value.Validate(); err != nil {
 					return err
 				}
 				return nil
@@ -106,12 +137,13 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "AddPage",
@@ -124,7 +156,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -197,7 +229,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 	u.RawQuery = q.Values().Encode()
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "POST", u, nil)
+	r, err := ht.NewRequest(ctx, "POST", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -228,24 +260,26 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 // GET /pages/{id}/file/{file_id}
 func (c *Client) GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error) {
 	res, err := c.sendGetFile(ctx, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res GetFileRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getFile"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages/{id}/file/{file_id}"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetFile",
@@ -258,7 +292,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -307,7 +341,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -335,24 +369,26 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 // GET /pages/{id}
 func (c *Client) GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error) {
 	res, err := c.sendGetPage(ctx, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res GetPageRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getPage"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages/{id}"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetPage",
@@ -365,7 +401,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -395,7 +431,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -423,24 +459,26 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 // GET /pages
 func (c *Client) GetPages(ctx context.Context) (Pages, error) {
 	res, err := c.sendGetPages(ctx)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getPages"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetPages",
@@ -453,7 +491,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -465,7 +503,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}

api/openapi/oas_handlers_gen.go

@@ -10,7 +10,8 @@ import (
 	"github.com/go-faster/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
-	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
 	"go.opentelemetry.io/otel/trace"
 
 	ht "github.com/ogen-go/ogen/http"
@@ -42,17 +43,18 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -89,10 +91,11 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 	var response AddPageRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "AddPage",
-			OperationID:   "addPage",
-			Body:          request,
+			Context:          ctx,
+			OperationName:    "AddPage",
+			OperationSummary: "Add new page",
+			OperationID:      "addPage",
+			Body:             request,
 			Params: middleware.Parameters{
 				{
 					Name: "url",
@@ -132,22 +135,27 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 		response, err = s.h.AddPage(ctx, request, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeAddPageResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -175,17 +183,18 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -207,10 +216,11 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 	var response GetFileRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "GetFile",
-			OperationID:   "getFile",
-			Body:          nil,
+			Context:          ctx,
+			OperationName:    "GetFile",
+			OperationSummary: "",
+			OperationID:      "getFile",
+			Body:             nil,
 			Params: middleware.Parameters{
 				{
 					Name: "id",
@@ -246,22 +256,27 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 		response, err = s.h.GetFile(ctx, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetFileResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -289,17 +304,18 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -321,10 +337,11 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 	var response GetPageRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "GetPage",
-			OperationID:   "getPage",
-			Body:          nil,
+			Context:          ctx,
+			OperationName:    "GetPage",
+			OperationSummary: "",
+			OperationID:      "getPage",
+			Body:             nil,
 			Params: middleware.Parameters{
 				{
 					Name: "id",
@@ -356,22 +373,27 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 		response, err = s.h.GetPage(ctx, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetPageResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -399,17 +421,18 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err error
 	)
@@ -417,12 +440,13 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 	var response Pages
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "GetPages",
-			OperationID:   "getPages",
-			Body:          nil,
-			Params:        middleware.Parameters{},
-			Raw:           r,
+			Context:          ctx,
+			OperationName:    "GetPages",
+			OperationSummary: "Get all pages",
+			OperationID:      "getPages",
+			Body:             nil,
+			Params:           middleware.Parameters{},
+			Raw:              r,
 		}
 
 		type (
@@ -447,22 +471,27 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 		response, err = s.h.GetPages(ctx)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetPagesResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }

api/openapi/oas_json_gen.go

@@ -23,12 +23,10 @@ func (s *AddPageBadRequest) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *AddPageBadRequest) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("field")
 		e.Str(s.Field)
 	}
 	{
-
 		e.FieldStart("error")
 		e.Str(s.Error)
 	}
@@ -138,7 +136,6 @@ func (s *AddPageReq) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *AddPageReq) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
@@ -280,7 +277,6 @@ func (s *Error) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Error) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("message")
 		e.Str(s.Message)
 	}
@@ -506,22 +502,18 @@ func (s *Page) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Page) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
 	{
-
 		e.FieldStart("created")
 		json.EncodeDateTime(e, s.Created)
 	}
 	{
-
 		e.FieldStart("formats")
 		e.ArrStart()
 		for _, elem := range s.Formats {
@@ -530,12 +522,10 @@ func (s *Page) encodeFields(e *jx.Encoder) {
 		e.ArrEnd()
 	}
 	{
-
 		e.FieldStart("status")
 		s.Status.Encode(e)
 	}
 	{
-
 		e.FieldStart("meta")
 		s.Meta.Encode(e)
 	}
@@ -699,12 +689,10 @@ func (s *PageMeta) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *PageMeta) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("title")
 		e.Str(s.Title)
 	}
 	{
-
 		e.FieldStart("description")
 		e.Str(s.Description)
 	}
@@ -831,22 +819,18 @@ func (s *PageWithResults) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *PageWithResults) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
 	{
-
 		e.FieldStart("created")
 		json.EncodeDateTime(e, s.Created)
 	}
 	{
-
 		e.FieldStart("formats")
 		e.ArrStart()
 		for _, elem := range s.Formats {
@@ -855,17 +839,14 @@ func (s *PageWithResults) encodeFields(e *jx.Encoder) {
 		e.ArrEnd()
 	}
 	{
-
 		e.FieldStart("status")
 		s.Status.Encode(e)
 	}
 	{
-
 		e.FieldStart("meta")
 		s.Meta.Encode(e)
 	}
 	{
-
 		e.FieldStart("results")
 		e.ArrStart()
 		for _, elem := range s.Results {
@@ -1052,12 +1033,10 @@ func (s *PageWithResultsMeta) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *PageWithResultsMeta) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("title")
 		e.Str(s.Title)
 	}
 	{
-
 		e.FieldStart("description")
 		e.Str(s.Description)
 	}
@@ -1234,7 +1213,6 @@ func (s *Result) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Result) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("format")
 		s.Format.Encode(e)
 	}
@@ -1245,7 +1223,6 @@ func (s *Result) encodeFields(e *jx.Encoder) {
 		}
 	}
 	{
-
 		e.FieldStart("files")
 		e.ArrStart()
 		for _, elem := range s.Files {
@@ -1374,22 +1351,18 @@ func (s *ResultFilesItem) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *ResultFilesItem) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("name")
 		e.Str(s.Name)
 	}
 	{
-
 		e.FieldStart("mimetype")
 		e.Str(s.Mimetype)
 	}
 	{
-
 		e.FieldStart("size")
 		e.Int64(s.Size)
 	}

api/openapi/oas_request_decoders_gen.go

@@ -77,9 +77,9 @@ func (s *Server) decodeAddPageRequest(r *http.Request) (
 			return req, close, err
 		}
 		if err := func() error {
-			if request.Set {
+			if value, ok := request.Get(); ok {
 				if err := func() error {
-					if err := request.Value.Validate(); err != nil {
+					if err := value.Validate(); err != nil {
 						return err
 					}
 					return nil

api/openapi/oas_request_encoders_gen.go

@@ -20,7 +20,7 @@ func encodeAddPageRequest(
 		// Keep request with empty body if value is not set.
 		return nil
 	}
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	{
 		if req.Set {
 			req.Encode(e)

api/openapi/oas_response_decoders_gen.go

@@ -15,7 +15,7 @@ import (
 	"github.com/ogen-go/ogen/validate"
 )
 
-func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
+func decodeAddPageResponse(resp *http.Response) (res AddPageRes, _ error) {
 	switch resp.StatusCode {
 	case 201:
 		// Code 201.
@@ -128,12 +128,12 @@ func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
+func decodeGetFileResponse(resp *http.Response) (res GetFileRes, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -216,12 +216,12 @@ func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
+func decodeGetPageResponse(resp *http.Response) (res GetPageRes, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -302,12 +302,12 @@ func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
+func decodeGetPagesResponse(resp *http.Response) (res Pages, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -385,7 +385,7 @@ func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }

api/openapi/oas_response_encoders_gen.go

@@ -10,6 +10,8 @@ import (
 	"github.com/go-faster/jx"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
+
+	ht "github.com/ogen-go/ogen/http"
 )
 
 func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trace.Span) error {
@@ -19,11 +21,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(201)
 		span.SetStatus(codes.Ok, http.StatusText(201))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *AddPageBadRequest:
@@ -31,11 +34,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(400)
 		span.SetStatus(codes.Error, http.StatusText(400))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	default:
@@ -54,6 +58,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileOKTextHTML:
@@ -65,6 +70,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileOKTextPlain:
@@ -76,6 +82,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileNotFound:
@@ -96,11 +103,12 @@ func encodeGetPageResponse(response GetPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(200)
 		span.SetStatus(codes.Ok, http.StatusText(200))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetPageNotFound:
@@ -119,11 +127,12 @@ func encodeGetPagesResponse(response Pages, w http.ResponseWriter, span trace.Sp
 	w.WriteHeader(200)
 	span.SetStatus(codes.Ok, http.StatusText(200))
 
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	response.Encode(e)
 	if _, err := e.WriteTo(w); err != nil {
 		return errors.Wrap(err, "write")
 	}
+
 	return nil
 }
 
@@ -142,11 +151,15 @@ func encodeErrorResponse(response *ErrorStatusCode, w http.ResponseWriter, span
 		span.SetStatus(codes.Ok, st)
 	}
 
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	response.Response.Encode(e)
 	if _, err := e.WriteTo(w); err != nil {
 		return errors.Wrap(err, "write")
 	}
+
+	if code >= http.StatusInternalServerError {
+		return errors.Wrapf(ht.ErrInternalServerErrorResponse, "code: %d, message: %s", code, http.StatusText(code))
+	}
 	return nil
 
 }

api/openapi/oas_router_gen.go

@@ -10,6 +10,19 @@ import (
 	"github.com/ogen-go/ogen/uri"
 )
 
+func (s *Server) cutPrefix(path string) (string, bool) {
+	prefix := s.cfg.Prefix
+	if prefix == "" {
+		return path, true
+	}
+	if !strings.HasPrefix(path, prefix) {
+		// Prefix doesn't match.
+		return "", false
+	}
+	// Cut prefix from the path.
+	return strings.TrimPrefix(path, prefix), true
+}
+
 // ServeHTTP serves http request as defined by OpenAPI v3 specification,
 // calling handler that matches the path or returning not found error.
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -21,17 +34,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			elemIsEscaped = strings.ContainsRune(elem, '%')
 		}
 	}
-	if prefix := s.cfg.Prefix; len(prefix) > 0 {
-		if strings.HasPrefix(elem, prefix) {
-			// Cut prefix from the path.
-			elem = strings.TrimPrefix(elem, prefix)
-		} else {
-			// Prefix doesn't match.
-			s.notFound(w, r)
-			return
-		}
-	}
-	if len(elem) == 0 {
+
+	elem, ok := s.cutPrefix(elem)
+	if !ok || len(elem) == 0 {
 		s.notFound(w, r)
 		return
 	}
@@ -129,6 +134,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 // Route is route object.
 type Route struct {
 	name        string
+	summary     string
 	operationID string
 	pathPattern string
 	count       int
@@ -142,6 +148,11 @@ func (r Route) Name() string {
 	return r.name
 }
 
+// Summary returns OpenAPI summary.
+func (r Route) Summary() string {
+	return r.summary
+}
+
 // OperationID returns OpenAPI operationId.
 func (r Route) OperationID() string {
 	return r.operationID
@@ -183,6 +194,11 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 		}()
 	}
 
+	elem, ok := s.cutPrefix(elem)
+	if !ok {
+		return r, false
+	}
+
 	// Static code generated router with unwrapped path search.
 	switch {
 	default:
@@ -201,6 +217,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 				switch method {
 				case "GET":
 					r.name = "GetPages"
+					r.summary = "Get all pages"
 					r.operationID = "getPages"
 					r.pathPattern = "/pages"
 					r.args = args
@@ -208,6 +225,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 					return r, true
 				case "POST":
 					r.name = "AddPage"
+					r.summary = "Add new page"
 					r.operationID = "addPage"
 					r.pathPattern = "/pages"
 					r.args = args
@@ -238,6 +256,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 					switch method {
 					case "GET":
 						r.name = "GetPage"
+						r.summary = ""
 						r.operationID = "getPage"
 						r.pathPattern = "/pages/{id}"
 						r.args = args
@@ -265,6 +284,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 						case "GET":
 							// Leaf: GetFile
 							r.name = "GetFile"
+							r.summary = ""
 							r.operationID = "getFile"
 							r.pathPattern = "/pages/{id}/file/{file_id}"
 							r.args = args

api/openapi/oas_schemas_gen.go

@@ -140,6 +140,16 @@ const (
 	FormatHeaders    Format = "headers"
 )
 
+// AllValues returns all Format values.
+func (Format) AllValues() []Format {
+	return []Format{
+		FormatAll,
+		FormatPdf,
+		FormatSingleFile,
+		FormatHeaders,
+	}
+}
+
 // MarshalText implements encoding.TextMarshaler.
 func (s Format) MarshalText() ([]byte, error) {
 	switch s {
@@ -189,6 +199,9 @@ type GetFileOKApplicationPdf struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKApplicationPdf) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -202,6 +215,9 @@ type GetFileOKTextHTML struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKTextHTML) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -215,6 +231,9 @@ type GetFileOKTextPlain struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKTextPlain) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -642,6 +661,17 @@ const (
 	StatusWithErrors Status = "with_errors"
 )
 
+// AllValues returns all Status values.
+func (Status) AllValues() []Status {
+	return []Status{
+		StatusNew,
+		StatusProcessing,
+		StatusDone,
+		StatusFailed,
+		StatusWithErrors,
+	}
+}
+
 // MarshalText implements encoding.TextMarshaler.
 func (s Status) MarshalText() ([]byte, error) {
 	switch s {

api/openapi/oas_validators_gen.go

@@ -42,6 +42,7 @@ func (s *AddPageReq) Validate() error {
 	}
 	return nil
 }
+
 func (s Format) Validate() error {
 	switch s {
 	case "all":
@@ -103,6 +104,7 @@ func (s *Page) Validate() error {
 	}
 	return nil
 }
+
 func (s *PageWithResults) Validate() error {
 	var failures []validate.FieldError
 	if err := func() error {
@@ -177,12 +179,14 @@ func (s *PageWithResults) Validate() error {
 	}
 	return nil
 }
+
 func (s Pages) Validate() error {
-	if s == nil {
+	alias := ([]Page)(s)
+	if alias == nil {
 		return errors.New("nil is invalid value")
 	}
 	var failures []validate.FieldError
-	for i, elem := range s {
+	for i, elem := range alias {
 		if err := func() error {
 			if err := elem.Validate(); err != nil {
 				return err
@@ -200,6 +204,7 @@ func (s Pages) Validate() error {
 	}
 	return nil
 }
+
 func (s *Result) Validate() error {
 	var failures []validate.FieldError
 	if err := func() error {
@@ -229,6 +234,7 @@ func (s *Result) Validate() error {
 	}
 	return nil
 }
+
 func (s Status) Validate() error {
 	switch s {
 	case "new":

application/application.go

@@ -12,11 +12,11 @@ import (
 
 	"github.com/dgraph-io/badger/v4"
 	"github.com/ogen-go/ogen/middleware"
-	"go.uber.org/multierr"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
 	"github.com/derfenix/webarchive/adapters/processors"
+	"github.com/derfenix/webarchive/adapters/repository"
 	badgerRepo "github.com/derfenix/webarchive/adapters/repository/badger"
 	"github.com/derfenix/webarchive/api/openapi"
 	"github.com/derfenix/webarchive/config"
@@ -30,7 +30,7 @@ func NewApplication(cfg config.Config) (Application, error) {
 		return Application{}, fmt.Errorf("new logger: %w", err)
 	}
 
-	db, err := badgerRepo.NewBadger(cfg.DB.Path, log.Named("db"))
+	db, err := repository.NewBadger(cfg.DB.Path, log.Named("db"))
 	if err != nil {
 		return Application{}, fmt.Errorf("new badger: %w", err)
 	}
@@ -40,7 +40,7 @@ func NewApplication(cfg config.Config) (Application, error) {
 		return Application{}, fmt.Errorf("new page repo: %w", err)
 	}
 
-	processor, err := processors.NewProcessors(cfg)
+	processor, err := processors.NewProcessors(cfg, log.Named("processor"))
 	if err != nil {
 		return Application{}, fmt.Errorf("new processors: %w", err)
 	}
@@ -170,15 +170,15 @@ func (a *Application) Stop() error {
 	var errs error
 
 	if err := a.db.Sync(); err != nil {
-		errs = multierr.Append(errs, fmt.Errorf("sync db: %w", err))
+		errs = errors.Join(errs, fmt.Errorf("sync db: %w", err))
 	}
 
-	if err := badgerRepo.Backup(a.db, badgerRepo.BackupStop); err != nil {
-		errs = multierr.Append(errs, fmt.Errorf("backup on stop: %w", err))
+	if err := repository.Backup(a.db, repository.BackupStop); err != nil {
+		errs = errors.Join(errs, fmt.Errorf("backup on stop: %w", err))
 	}
 
 	if err := a.db.Close(); err != nil {
-		errs = multierr.Append(errs, fmt.Errorf("close db: %w", err))
+		errs = errors.Join(errs, fmt.Errorf("close db: %w", err))
 	}
 
 	return errs
@@ -189,6 +189,7 @@ func newLogger(cfg config.Logging) (*zap.Logger, error) {
 	logCfg.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder
 	logCfg.EncoderConfig.EncodeDuration = zapcore.NanosDurationEncoder
 	logCfg.DisableCaller = true
+	logCfg.DisableStacktrace = true
 
 	logCfg.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
 	if cfg.Debug {

cmd/service/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/signal"
 	"sync"
+	"syscall"
 
 	"go.uber.org/zap"
 
@@ -14,7 +15,7 @@ import (
 )
 
 func main() {
-	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill)
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer cancel()
 
 	cfg, err := config.NewConfig(ctx)

config/config.go

@@ -17,7 +17,10 @@ func NewConfig(ctx context.Context) (Config, error) {
 		envconfig.OsLookuper(),
 	)
 
-	if err := envconfig.ProcessWith(ctx, &cfg, lookuper); err != nil {
+	if err := envconfig.ProcessWith(ctx, &envconfig.Config{
+		Target:   &cfg,
+		Lookuper: lookuper,
+	}); err != nil {
 		return Config{}, fmt.Errorf("process env: %w", err)
 	}
 

devenv.lock

@@ -0,0 +1,119 @@
+{
+  "nodes": {
+    "devenv": {
+      "locked": {
+        "dir": "src/modules",
+        "lastModified": 1741670053,
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "47abb5dfd5b7824a0979ef86f3986aea48847312",
+        "type": "github"
+      },
+      "original": {
+        "dir": "src/modules",
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1733328505,
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "git-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1741379162,
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "rev": "b5a62751225b2f62ff3147d0a334055ebadcd5cc",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "git-hooks.nix",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "git-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1733477122,
+        "owner": "cachix",
+        "repo": "devenv-nixpkgs",
+        "rev": "7bd9e84d0452f6d2e63b6e6da29fe73fac951857",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "rolling",
+        "repo": "devenv-nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_stable": {
+      "locked": {
+        "lastModified": 1741886019,
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "ed218927a5be8252112adf70905f2a282e2f6692",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "release-24.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "git-hooks": "git-hooks",
+        "nixpkgs": "nixpkgs",
+        "nixpkgs_stable": "nixpkgs_stable",
+        "pre-commit-hooks": [
+          "git-hooks"
+        ]
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

devenv.nix

@@ -0,0 +1,22 @@
+{
+  pkgs,
+  ...
+}:
+
+{
+
+  packages = [
+    pkgs.git
+    pkgs.go_1_23
+  ];
+
+  enterShell = ''
+    git --version
+    go version
+  '';
+
+  enterTest = ''
+    echo "Running tests"
+    go test -race ./...
+  '';
+}

devenv.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json
+inputs:
+  nixpkgs:
+    url: github:cachix/devenv-nixpkgs/rolling
+  nixpkgs_stable:
+    url: "github:NixOS/nixpkgs/release-24.11"
+# If you're using non-OSS software, you can set allowUnfree to true.
+# allowUnfree: true
+
+# If you're willing to use a package that's vulnerable
+# permittedInsecurePackages:
+#  - "openssl-1.1.1w"
+
+# If you have more than one devenv you can merge them
+#imports:
+# - ./backend

entity/cache.go

@@ -0,0 +1,42 @@
+package entity
+
+import (
+	"bytes"
+	"io"
+	"sync"
+)
+
+func NewCache() *Cache {
+	return &Cache{data: make([]byte, 0, 1024*512)}
+}
+
+type Cache struct {
+	mu   sync.RWMutex
+	data []byte
+}
+
+func (c *Cache) Write(p []byte) (n int, err error) {
+	c.mu.Lock()
+	c.data = append(c.data, p...)
+	c.mu.Unlock()
+
+	return len(p), nil
+}
+
+func (c *Cache) Get() []byte {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.data
+}
+
+func (c *Cache) Reader() io.Reader {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if len(c.data) == 0 {
+		return nil
+	}
+
+	return bytes.NewBuffer(c.data)
+}

entity/page.go

@@ -3,6 +3,7 @@ package entity
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -10,8 +11,8 @@ import (
 )
 
 type Processor interface {
-	Process(ctx context.Context, format Format, url string) Result
-	GetMeta(ctx context.Context, url string) (Meta, error)
+	Process(ctx context.Context, format Format, url string, cache *Cache) Result
+	GetMeta(ctx context.Context, url string, cache *Cache) (Meta, error)
 }
 
 type Format uint8
@@ -41,59 +42,79 @@ const (
 type Meta struct {
 	Title       string
 	Description string
+	Encoding    string
 	Error       string
 }
 
-func NewPage(url string, description string, formats ...Format) *Page {
-	return &Page{
-		ID:          uuid.New(),
-		URL:         url,
-		Description: description,
-		Formats:     formats,
-		Created:     time.Now(),
-		Version:     1,
-	}
-}
-
-type Page struct {
+type PageBase struct {
 	ID          uuid.UUID
 	URL         string
 	Description string
 	Created     time.Time
 	Formats     []Format
-	Results     Results
 	Version     uint16
 	Status      Status
 	Meta        Meta
 }
 
+func NewPage(url string, description string, formats ...Format) *Page {
+	return &Page{
+		PageBase: PageBase{
+			ID:          uuid.New(),
+			URL:         url,
+			Description: description,
+			Formats:     formats,
+			Created:     time.Now(),
+			Version:     1,
+		},
+		cache: NewCache(),
+	}
+}
+
+type Page struct {
+	PageBase
+	Results ResultsRO
+	cache   *Cache
+}
+
 func (p *Page) SetProcessing() {
 	p.Status = StatusProcessing
 }
 
+func (p *Page) Prepare(ctx context.Context, processor Processor) {
+	meta, err := processor.GetMeta(ctx, p.URL, p.cache)
+	if err != nil {
+		p.Meta.Error = err.Error()
+	} else {
+		p.Meta = meta
+	}
+}
+
 func (p *Page) Process(ctx context.Context, processor Processor) {
 	innerWG := sync.WaitGroup{}
 	innerWG.Add(len(p.Formats))
 
+	results := Results{}
+
 	for _, format := range p.Formats {
 		go func(format Format) {
 			defer innerWG.Done()
 
 			defer func() {
 				if err := recover(); err != nil {
-					p.Results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v", err)})
+					results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v (%s)", err, string(debug.Stack()))})
 				}
 			}()
 
-			result := processor.Process(ctx, format, p.URL)
-			p.Results.Add(result)
+			result := processor.Process(ctx, format, p.URL, p.cache)
+			results.Add(result)
 		}(format)
 	}
 
 	innerWG.Wait()
 
 	var hasResultWithOutErrors bool
-	for _, result := range p.Results.Results() {
+	for _, result := range results.Results() {
 		if result.Err != nil {
 			p.Status = StatusWithErrors
 		} else {
@@ -108,4 +129,6 @@ func (p *Page) Process(ctx context.Context, processor Processor) {
 	if p.Status == StatusProcessing {
 		p.Status = StatusDone
 	}
+
+	p.Results = results.RO()
 }

entity/result.go

@@ -6,6 +6,8 @@ import (
 	"github.com/vmihailenco/msgpack/v5"
 )
 
+type ResultsRO []Result
+
 type Result struct {
 	Format Format
 	Err    error
@@ -17,6 +19,13 @@ type Results struct {
 	results []Result
 }
 
+func (r *Results) RO() ResultsRO {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.results
+}
+
 func (r *Results) MarshalMsgpack() ([]byte, error) {
 	return msgpack.Marshal(r.results)
 }

entity/worker.go

@@ -9,7 +9,7 @@ import (
 
 type Pages interface {
 	Save(ctx context.Context, page *Page) error
-	ListUnprocessed(ctx context.Context) ([]*Page, error)
+	ListUnprocessed(ctx context.Context) ([]Page, error)
 }
 
 func NewWorker(ch chan *Page, pages Pages, processor Processor, log *zap.Logger) *Worker {
@@ -37,7 +37,7 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
 			w.log.Error("failed to get unprocessed pages", zap.Error(err))
 		} else {
 			for i := range unprocessed {
-				w.ch <- unprocessed[i]
+				w.ch <- &unprocessed[i]
 			}
 		}
 	}()
@@ -66,6 +66,16 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
 func (w *Worker) do(ctx context.Context, wg *sync.WaitGroup, page *Page, log *zap.Logger) {
 	defer wg.Done()
 
+	page.SetProcessing()
+	if err := w.pages.Save(ctx, page); err != nil {
+		w.log.Error(
+			"failed to save processing page",
+			zap.String("page_id", page.ID.String()),
+			zap.String("page_url", page.URL),
+			zap.Error(err),
+		)
+	}
+
 	page.Process(ctx, w.processor)
 
 	log.Debug("page processed")

go.mod

@@ -1,59 +1,63 @@
 module github.com/derfenix/webarchive
 
-go 1.19
+go 1.23.0
+
+toolchain go1.23.6
 
 require (
-	github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0
-	github.com/dgraph-io/badger/v4 v4.0.1
-	github.com/gabriel-vasile/mimetype v1.4.2
-	github.com/go-faster/errors v0.6.1
-	github.com/go-faster/jx v1.0.0
-	github.com/google/uuid v1.3.0
-	github.com/ogen-go/ogen v0.60.1
-	github.com/sethvargo/go-envconfig v0.9.0
-	github.com/stretchr/testify v1.8.2
-	github.com/vmihailenco/msgpack/v5 v5.3.5
-	go.opentelemetry.io/otel v1.14.0
-	go.opentelemetry.io/otel/metric v0.37.0
-	go.opentelemetry.io/otel/trace v1.14.0
-	go.uber.org/multierr v1.10.0
-	go.uber.org/zap v1.24.0
+	github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.3
+	github.com/dgraph-io/badger/v4 v4.6.0
+	github.com/disintegration/imaging v1.6.2
+	github.com/gabriel-vasile/mimetype v1.4.8
+	github.com/go-faster/errors v0.7.1
+	github.com/go-faster/jx v1.1.0
+	github.com/google/uuid v1.6.0
+	github.com/minio/minio-go/v7 v7.0.88
+	github.com/ogen-go/ogen v1.10.1
+	github.com/sethvargo/go-envconfig v1.1.1
+	github.com/stretchr/testify v1.10.0
+	github.com/vmihailenco/msgpack/v5 v5.4.1
+	go.opentelemetry.io/otel v1.35.0
+	go.opentelemetry.io/otel/metric v1.35.0
+	go.opentelemetry.io/otel/trace v1.35.0
+	go.uber.org/multierr v1.11.0
+	go.uber.org/zap v1.27.0
+	golang.org/x/net v0.38.0
 )
 
 require (
-	github.com/benbjohnson/clock v1.3.0 // indirect
-	github.com/cespare/xxhash v1.1.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgraph-io/ristretto v0.1.1 // indirect
-	github.com/dlclark/regexp2 v1.8.1 // indirect
+	github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect
+	github.com/dlclark/regexp2 v1.11.5 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/fatih/color v1.18.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-faster/yamlx v0.4.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-faster/yaml v0.4.6 // indirect
+	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/glog v1.1.1 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v23.3.3+incompatible // indirect
-	github.com/klauspost/compress v1.16.3 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/google/flatbuffers v25.2.10+incompatible // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/crc64nvme v1.0.1 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rs/xid v1.6.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
-	go.opencensus.io v0.24.0 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
+	golang.org/x/image v0.25.0 // indirect
+	golang.org/x/sync v0.12.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
+	google.golang.org/protobuf v1.36.5 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,231 +1,124 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0 h1:DNrExYwvyyI404SxdUCCANAj9TwnGjRfa3cYFMNY1AU=
-github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0/go.mod h1:SQq4xfIdvf6WYKSDxAJc+xOJdolt+/bc1jnQKMtPMvQ=
-github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
-github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.3 h1:vrA6+R1BMLKMTbos8jAeuBrImHPGtY4gTlcue3OIej8=
+github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.3/go.mod h1:SQq4xfIdvf6WYKSDxAJc+xOJdolt+/bc1jnQKMtPMvQ=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/badger/v4 v4.0.1 h1:zwLYFc4sfxKdaRTvS6wlHsSuYWNUiWnYLU+TS+/nCDI=
-github.com/dgraph-io/badger/v4 v4.0.1/go.mod h1:edFJfgVfwYjg+grodpS7Yj2vohQMK3VL6eCaR6EpRJU=
-github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
-github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0=
-github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dgraph-io/badger/v4 v4.6.0 h1:acOwfOOZ4p1dPRnYzvkVm7rUk2Y21TgPVepCy5dJdFQ=
+github.com/dgraph-io/badger/v4 v4.6.0/go.mod h1:KSJ5VTuZNC3Sd+YhvVjk2nYua9UZnnTr/SkXvdtiPgI=
+github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I=
+github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
-github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
-github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
-github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
+github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI=
-github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY=
-github.com/go-faster/jx v1.0.0 h1:HE+ms2e6ZGkZ6u13t8u+onBinrPvIPI+0hWXGELm74g=
-github.com/go-faster/jx v1.0.0/go.mod h1:zm8SlkwK+H0TYNKYtVJ/7cWFS7soJBQWhcPctKyYL/4=
-github.com/go-faster/yamlx v0.4.1 h1:00RQkZopoLDF1SgBDJVHuN6epTOK7T0TkN427vbvEBk=
-github.com/go-faster/yamlx v0.4.1/go.mod h1:QXr/i3Z00jRhskgyWkoGsEdseebd/ZbZEpGS6DJv8oo=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
+github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg=
+github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg=
+github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
+github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.1.1 h1:jxpi2eWoU84wbX9iIEyAeeoac3FLuifZpY9tcNUD9kw=
-github.com/golang/glog v1.1.1/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
-github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
-github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
-github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
-github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
-github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
-github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/flatbuffers v23.3.3+incompatible h1:5PJI/WbJkaMTvpGxsHVKG/LurN/KnWXNyGpwSCDgen0=
-github.com/google/flatbuffers v23.3.3+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
-github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
+github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
+github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/ogen-go/ogen v0.60.1 h1:yOt0i6NcH7jM3rBi9nnv5VsGUQRw4ACUMsiJojnqrAM=
-github.com/ogen-go/ogen v0.60.1/go.mod h1:tcwLpHe4vyk9xtbTMe3yu3Qtcbz8VjrpBz9LzsdwWvQ=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY=
+github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.88 h1:v8MoIJjwYxOkehp+eiLIuvXk87P2raUtoU5klrAAshs=
+github.com/minio/minio-go/v7 v7.0.88/go.mod h1:33+O8h0tO7pCeCWwBVa07RhVVfB/3vS4kEX7rwYKmIg=
+github.com/ogen-go/ogen v1.10.1 h1:oeSN8AF9mhTVfapbMuL8pQTF2ToqyW9xXaStmOhHKTA=
+github.com/ogen-go/ogen v1.10.1/go.mod h1:fXCg9PsNYEzJ8ABdmZ2A7j4hMi9EDHP53jzsNtIM3d0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
-github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE=
-github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
-github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/sethvargo/go-envconfig v1.1.1 h1:JDu8Q9baIzJf47NPkzhIB6aLYL0vQ+pPypoYrejS9QY=
+github.com/sethvargo/go-envconfig v1.1.1/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
+github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
-go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
-go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
-go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
-go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
-go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
-go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
-go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
+go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
+go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M=
+go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
+go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
+go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
+golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw=
+golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
+golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
+golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
+golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
-google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
-google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
-google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
-google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
-google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
+google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
+google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

ports/rest/converter.go

@@ -29,10 +29,10 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults {
 			Error:       openapi.NewOptString(page.Meta.Error),
 		},
 		Results: func() []openapi.Result {
-			results := make([]openapi.Result, len(page.Results.Results()))
+			results := make([]openapi.Result, len(page.Results))
 
 			for i := range results {
-				result := &(page.Results.Results())[i]
+				result := &page.Results[i]
 
 				errText := openapi.OptString{}
 				if result.Err != nil {
@@ -66,6 +66,29 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults {
 	}
 }
 
+func BasePageToRest(page *entity.PageBase) openapi.Page {
+	return openapi.Page{
+		ID:      page.ID,
+		URL:     page.URL,
+		Created: page.Created,
+		Meta: openapi.PageMeta{
+			Title:       html.EscapeString(page.Meta.Title),
+			Description: html.EscapeString(page.Meta.Description),
+			Error:       openapi.NewOptString(page.Meta.Error),
+		},
+		Formats: func() []openapi.Format {
+			res := make([]openapi.Format, len(page.Formats))
+
+			for i, format := range page.Formats {
+				res[i] = FormatToRest(format)
+			}
+
+			return res
+		}(),
+		Status: StatusToRest(page.Status),
+	}
+}
+
 func PageToRest(page *entity.Page) openapi.Page {
 	return openapi.Page{
 		ID:      page.ID,

ports/rest/service.go

@@ -30,9 +30,9 @@ func NewService(pages Pages, ch chan *entity.Page, processor entity.Processor) *
 
 type Service struct {
 	openapi.UnimplementedHandler
+	processor entity.Processor
 	pages     Pages
 	ch        chan *entity.Page
-	processor entity.Processor
 }
 
 func (s *Service) GetPage(ctx context.Context, params openapi.GetPageParams) (openapi.GetPageRes, error) {
@@ -81,20 +81,14 @@ func (s *Service) AddPage(ctx context.Context, req openapi.OptAddPageReq, params
 	}
 
 	page := entity.NewPage(url, description, domainFormats...)
-	page.Status = entity.StatusProcessing
-
-	meta, err := s.processor.GetMeta(ctx, page.URL)
-	if err != nil {
-		page.Meta.Error = err.Error()
-	} else {
-		page.Meta = meta
-	}
+	page.Status = entity.StatusNew
+	page.Prepare(ctx, s.processor)
 
 	if err := s.pages.Save(ctx, page); err != nil {
 		return nil, fmt.Errorf("save page: %w", err)
 	}
 
-	res := PageToRest(page)
+	res := BasePageToRest(&page.PageBase)
 
 	s.ch <- page