Merge branch 'master' into dependabot/go_modules/google.golang.org/protobuf-1.33.0

Bump golang.org/x/image from 0.10.0 to 0.18.0

.github/workflows/release.yaml

@@ -1,6 +1,7 @@
+---
 name: release
 
-on:
+"on":
   push:
     tags:
       - 'v*'

.github/workflows/test.yaml

@@ -1,6 +1,7 @@
+---
 name: test
 
-on:
+"on":
   pull_request:
   push:
     branches:

.gitignore

@@ -43,3 +43,5 @@ fabric.properties
 go.work
 test.http
 db
+http-client.env.json
+http-client.private.env.json

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

.idea/jsonSchemas.xml

@@ -3,11 +3,11 @@
   <component name="JsonSchemaMappingsProjectConfiguration">
     <state>
       <map>
-        <entry key="openapi">
+        <entry key="OpenAPI 3.0">
           <value>
             <SchemaInfo>
-              <option name="name" value="openapi" />
-              <option name="relativePathToSchema" value="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json" />
+              <option name="name" value="OpenAPI 3.0" />
+              <option name="relativePathToSchema" value="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/schemas/v3.0/schema.json" />
               <option name="applicationDefined" value="true" />
               <option name="patterns">
                 <list>

.idea/swagger-settings.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SwaggerSettings">
+    <option name="defaultPreviewType" value="SWAGGER_UI" />
+  </component>
+</project>

.idea/yamllint.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="YamllintSettings">
+    <enabled>true</enabled>
+    <binPath>/usr/bin/yamllint</binPath>
+  </component>
+</project>

README.md

@@ -116,3 +116,4 @@ curl -X GET --location "http://localhost:5001/api/v1/pages" | jq .
 - [ ] Multi-user access
 - [ ] Support SQL database with or without separate files storage
 - [ ] Tags/Categories
+- [ ] Save page to markdown

adapters/processors/headers.go

@@ -17,7 +17,7 @@ type Headers struct {
 	client *http.Client
 }
 
-func (h *Headers) Process(ctx context.Context, url string) ([]entity.File, error) {
+func (h *Headers) Process(ctx context.Context, url string, _ *entity.Cache) ([]entity.File, error) {
 	var (
 		headersFile entity.File
 		err         error

adapters/processors/internal/mediainline.go

@@ -0,0 +1,255 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/disintegration/imaging"
+	"github.com/gabriel-vasile/mimetype"
+	"go.uber.org/zap"
+	"golang.org/x/net/html"
+)
+
+type MediaInline struct {
+	log    *zap.Logger
+	getter func(context.Context, string) (*http.Response, error)
+}
+
+func NewMediaInline(log *zap.Logger, getter func(context.Context, string) (*http.Response, error)) *MediaInline {
+	return &MediaInline{log: log, getter: getter}
+}
+
+func (m *MediaInline) Inline(ctx context.Context, reader io.Reader, pageURL string) (*html.Node, error) {
+	htmlNode, err := html.Parse(reader)
+	if err != nil {
+		return nil, fmt.Errorf("parse response body: %w", err)
+	}
+
+	baseURL, err := url.Parse(pageURL)
+	if err != nil {
+		return nil, fmt.Errorf("parse page url: %w", err)
+	}
+
+	m.visit(ctx, htmlNode, m.processorFunc, baseURL)
+
+	return htmlNode, nil
+}
+
+func (m *MediaInline) processorFunc(ctx context.Context, node *html.Node, baseURL *url.URL) error {
+	switch node.Data {
+	case "link":
+		if err := m.processHref(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process link %s: %w", node.Attr, err)
+		}
+
+	case "script", "img":
+		if err := m.processSrc(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process script %s: %w", node.Attr, err)
+		}
+
+	case "a":
+		if err := m.processAHref(node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process a href %s: %w", node.Attr, err)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processAHref(attrs []html.Attribute, baseURL *url.URL) error {
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "href":
+			attrs[idx].Val = normalizeURL(attr.Val, baseURL)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processHref(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "rel":
+			switch attr.Val {
+			case "stylesheet", "icon", "alternate icon", "shortcut icon", "manifest":
+				shouldProcess = true
+			}
+
+		case "href":
+			value = attr.Val
+			valueIdx = idx
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) processSrc(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "src":
+			value = attr.Val
+			valueIdx = idx
+			shouldProcess = true
+		case "data-src":
+			value = attr.Val
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) loadAndEncode(ctx context.Context, baseURL *url.URL, value string) (string, error) {
+	mime := "text/plain"
+
+	if value == "" {
+		return "", nil
+	}
+
+	normalizedURL := normalizeURL(value, baseURL)
+	if normalizedURL == "" {
+		return value, nil
+	}
+
+	response, err := m.getter(ctx, normalizedURL)
+	if err != nil {
+		m.log.Sugar().With(zap.Error(err)).Errorf("load %s", normalizedURL)
+		return value, nil
+	}
+
+	defer func() {
+		_ = response.Body.Close()
+	}()
+
+	cleanMime := func(s string) string {
+		s, _, _ = strings.Cut(s, "+")
+		return s
+	}
+
+	if ct := response.Header.Get("Content-Type"); ct != "" {
+		mime = ct
+	}
+
+	encodedVal, err := m.encodeResource(response.Body, &mime)
+	if err != nil {
+		return value, fmt.Errorf("encode resource: %w", err)
+	}
+
+	return fmt.Sprintf("data:%s;base64, %s", cleanMime(mime), encodedVal), nil
+}
+
+func (m *MediaInline) visit(ctx context.Context, n *html.Node, proc func(context.Context, *html.Node, *url.URL) error, baseURL *url.URL) {
+	if err := proc(ctx, n, baseURL); err != nil {
+		m.log.Error("process error", zap.Error(err))
+	}
+
+	if n.FirstChild != nil {
+		m.visit(ctx, n.FirstChild, proc, baseURL)
+	}
+
+	if n.NextSibling != nil {
+		m.visit(ctx, n.NextSibling, proc, baseURL)
+	}
+}
+
+func normalizeURL(resourceURL string, base *url.URL) string {
+	if strings.HasPrefix(resourceURL, "//") {
+		return "https:" + resourceURL
+	}
+
+	if strings.HasPrefix(resourceURL, "about:") {
+		return ""
+	}
+
+	parsedResourceURL, err := url.Parse(resourceURL)
+	if err != nil {
+		return resourceURL
+	}
+
+	reference := base.ResolveReference(parsedResourceURL)
+
+	return reference.String()
+}
+
+func (m *MediaInline) encodeResource(r io.Reader, mime *string) (string, error) {
+	all, err := io.ReadAll(r)
+	if err != nil {
+		return "", fmt.Errorf("read data: %w", err)
+	}
+
+	all, err = m.preprocessResource(all, mime)
+	if err != nil {
+		return "", fmt.Errorf("preprocess resource: %w", err)
+	}
+
+	return base64.StdEncoding.EncodeToString(all), nil
+}
+
+func (m *MediaInline) preprocessResource(data []byte, mime *string) ([]byte, error) {
+	detectedMime := mimetype.Detect(data)
+
+	switch {
+	case strings.HasPrefix(detectedMime.String(), "image"):
+		decodedImage, err := imaging.Decode(bytes.NewBuffer(data))
+		if err != nil {
+			m.log.Error("failed to decode image", zap.Error(err))
+
+			return data, nil
+		}
+
+		if size := decodedImage.Bounds().Size(); size.X > 1024 || size.Y > 1024 {
+			thumbnail := imaging.Thumbnail(decodedImage, 1024, 1024, imaging.Lanczos)
+			buf := bytes.NewBuffer(nil)
+
+			if err := imaging.Encode(buf, thumbnail, imaging.JPEG, imaging.JPEGQuality(90)); err != nil {
+				m.log.Error("failed to create resized image", zap.Error(err))
+
+				return data, nil
+			}
+
+			*mime = "image/jpeg"
+			m.log.Info("Resized")
+
+			return buf.Bytes(), nil
+		}
+	}
+
+	return data, nil
+}

adapters/processors/pdf.go

@@ -19,7 +19,7 @@ type PDF struct {
 	cfg config.PDF
 }
 
-func (p *PDF) Process(_ context.Context, url string) ([]entity.File, error) {
+func (p *PDF) Process(_ context.Context, url string, cache *entity.Cache) ([]entity.File, error) {
 	gen, err := wkhtmltopdf.NewPDFGenerator()
 	if err != nil {
 		return nil, fmt.Errorf("new pdf generator: %w", err)
@@ -37,17 +37,29 @@ func (p *PDF) Process(_ context.Context, url string) ([]entity.File, error) {
 	gen.Grayscale.Set(p.cfg.Grayscale)
 	gen.Title.Set(url)
 
-	page := wkhtmltopdf.NewPage(url)
-	page.PrintMediaType.Set(p.cfg.MediaPrint)
-	page.JavascriptDelay.Set(200)
-	page.LoadMediaErrorHandling.Set("ignore")
-	page.FooterRight.Set("[page]")
-	page.HeaderLeft.Set(url)
-	page.HeaderRight.Set(time.Now().Format(time.DateOnly))
-	page.FooterFontSize.Set(10)
-	page.Zoom.Set(p.cfg.Zoom)
-	page.ViewportSize.Set(p.cfg.Viewport)
-	page.NoBackground.Set(true)
+	opts := wkhtmltopdf.NewPageOptions()
+	opts.PrintMediaType.Set(p.cfg.MediaPrint)
+	opts.JavascriptDelay.Set(200)
+	opts.DisableJavascript.Set(false)
+	opts.LoadErrorHandling.Set("ignore")
+	opts.LoadMediaErrorHandling.Set("skip")
+	opts.FooterRight.Set("[opts]")
+	opts.HeaderLeft.Set(url)
+	opts.HeaderRight.Set(time.Now().Format(time.DateOnly))
+	opts.FooterFontSize.Set(10)
+	opts.Zoom.Set(p.cfg.Zoom)
+	opts.ViewportSize.Set(p.cfg.Viewport)
+	opts.NoBackground.Set(true)
+	opts.DisableLocalFileAccess.Set(false)
+	opts.DisableExternalLinks.Set(false)
+	opts.DisableInternalLinks.Set(false)
+
+	var page wkhtmltopdf.PageProvider
+	if len(cache.Get()) > 0 {
+		page = &wkhtmltopdf.PageReader{Input: cache.Reader(), PageOptions: opts}
+	} else {
+		page = &wkhtmltopdf.Page{Input: url, PageOptions: opts}
+	}
 
 	gen.AddPage(page)
 

adapters/processors/processors.go

@@ -3,22 +3,27 @@ package processors
 import (
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
+	"strings"
 	"time"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/html"
 
 	"github.com/derfenix/webarchive/config"
 	"github.com/derfenix/webarchive/entity"
 )
 
+const defaultEncoding = "utf-8"
+
 type processor interface {
-	Process(ctx context.Context, url string) ([]entity.File, error)
+	Process(ctx context.Context, url string, cache *entity.Cache) ([]entity.File, error)
 }
 
-func NewProcessors(cfg config.Config) (*Processors, error) {
+func NewProcessors(cfg config.Config, log *zap.Logger) (*Processors, error) {
 	jar, err := cookiejar.New(&cookiejar.Options{
 		PublicSuffixList: nil,
 	})
@@ -58,7 +63,7 @@ func NewProcessors(cfg config.Config) (*Processors, error) {
 		processors: map[entity.Format]processor{
 			entity.FormatHeaders:    NewHeaders(httpClient),
 			entity.FormatPDF:        NewPDF(cfg.PDF),
-			entity.FormatSingleFile: NewSingleFile(httpClient),
+			entity.FormatSingleFile: NewSingleFile(httpClient, log),
 		},
 	}
 
@@ -70,7 +75,7 @@ type Processors struct {
 	client     *http.Client
 }
 
-func (p *Processors) Process(ctx context.Context, format entity.Format, url string) entity.Result {
+func (p *Processors) Process(ctx context.Context, format entity.Format, url string, cache *entity.Cache) entity.Result {
 	result := entity.Result{Format: format}
 
 	proc, ok := p.processors[format]
@@ -80,7 +85,7 @@ func (p *Processors) Process(ctx context.Context, format entity.Format, url stri
 		return result
 	}
 
-	files, err := proc.Process(ctx, url)
+	files, err := proc.Process(ctx, url, cache)
 	if err != nil {
 		result.Err = fmt.Errorf("process: %w", err)
 
@@ -98,7 +103,7 @@ func (p *Processors) OverrideProcessor(format entity.Format, proc processor) err
 	return nil
 }
 
-func (p *Processors) GetMeta(ctx context.Context, url string) (entity.Meta, error) {
+func (p *Processors) GetMeta(ctx context.Context, url string, cache *entity.Cache) (entity.Meta, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return entity.Meta{}, fmt.Errorf("new request: %w", err)
@@ -121,13 +126,37 @@ func (p *Processors) GetMeta(ctx context.Context, url string) (entity.Meta, erro
 		_ = response.Body.Close()
 	}()
 
-	htmlNode, err := html.Parse(response.Body)
+	tee := io.TeeReader(response.Body, cache)
+
+	htmlNode, err := html.Parse(tee)
 	if err != nil {
 		return entity.Meta{}, fmt.Errorf("parse response body: %w", err)
 	}
 
+	var fc *html.Node
+	for fc = htmlNode.FirstChild; fc != nil && fc.Data != "html"; fc = fc.NextSibling {
+	}
+
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
+	fc = fc.NextSibling
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
+	for fc = fc.FirstChild; fc != nil && fc.Data != "head"; fc = fc.NextSibling {
+		fmt.Println(fc.Data)
+	}
+
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
 	meta := entity.Meta{}
-	getMetaData(htmlNode, &meta)
+	getMetaData(fc, &meta)
+	meta.Encoding = encodingFromHeader(response.Header)
 
 	return meta, nil
 }
@@ -156,3 +185,19 @@ func getMetaData(n *html.Node, meta *entity.Meta) {
 		getMetaData(c, meta)
 	}
 }
+
+func encodingFromHeader(headers http.Header) string {
+	var foundEncoding bool
+	var encoding string
+
+	_, encoding, foundEncoding = strings.Cut(headers.Get("Content-Type"), "; ")
+	if foundEncoding {
+		_, encoding, foundEncoding = strings.Cut(encoding, "=")
+	}
+
+	if !foundEncoding {
+		encoding = defaultEncoding
+	}
+
+	return encoding
+}

adapters/processors/processors_test.go

@@ -0,0 +1,30 @@
+package processors
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+
+	"github.com/derfenix/webarchive/config"
+	"github.com/derfenix/webarchive/entity"
+)
+
+func TestProcessors_GetMeta(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	cfg, err := config.NewConfig(ctx)
+	require.NoError(t, err)
+
+	procs, err := NewProcessors(cfg, zaptest.NewLogger(t))
+	require.NoError(t, err)
+
+	cache := entity.NewCache()
+
+	meta, err := procs.GetMeta(ctx, "https://habr.com/ru/companies/wirenboard/articles/722718/", cache)
+	require.NoError(t, err)
+	assert.Equal(t, "Сколько стоит умный дом? Рассказываю, как строил свой и что получилось за 1000 руб./м² / Хабр", meta.Title)
+}

adapters/processors/singlefile.go

@@ -3,30 +3,57 @@ package processors
 import (
 	"bytes"
 	"context"
-	"encoding/base64"
 	"fmt"
-	"io"
 	"net/http"
-	"net/url"
-	"strings"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/html"
-	"golang.org/x/net/html/atom"
 
+	"github.com/derfenix/webarchive/adapters/processors/internal"
 	"github.com/derfenix/webarchive/entity"
 )
 
-const defaultEncoding = "utf-8"
-
-func NewSingleFile(client *http.Client) *SingleFile {
-	return &SingleFile{client: client}
+func NewSingleFile(client *http.Client, log *zap.Logger) *SingleFile {
+	return &SingleFile{client: client, log: log}
 }
 
 type SingleFile struct {
 	client *http.Client
+	log    *zap.Logger
 }
 
-func (s *SingleFile) Process(ctx context.Context, url string) ([]entity.File, error) {
+func (s *SingleFile) Process(ctx context.Context, pageURL string, cache *entity.Cache) ([]entity.File, error) {
+	reader := cache.Reader()
+
+	if reader == nil {
+		response, err := s.get(ctx, pageURL)
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			_ = response.Body.Close()
+		}()
+
+		reader = response.Body
+	}
+
+	inlinedHTML, err := internal.NewMediaInline(s.log, s.get).Inline(ctx, reader, pageURL)
+	if err != nil {
+		return nil, fmt.Errorf("inline media: %w", err)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	if err := html.Render(buf, inlinedHTML); err != nil {
+		return nil, fmt.Errorf("render result html: %w", err)
+	}
+
+	htmlFile := entity.NewFile("page.html", buf.Bytes())
+
+	return []entity.File{htmlFile}, nil
+}
+
+func (s *SingleFile) get(ctx context.Context, url string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("new request: %w", err)
@@ -45,180 +72,5 @@ func (s *SingleFile) Process(ctx context.Context, url string) ([]entity.File, er
 		return nil, fmt.Errorf("empty response body")
 	}
 
-	defer func() {
-		_ = response.Body.Close()
-	}()
-
-	htmlNode, err := html.Parse(response.Body)
-	if err != nil {
-		return nil, fmt.Errorf("parse response body: %w", err)
-	}
-
-	if err := s.crawl(ctx, htmlNode, baseURL(url), getEncoding(response)); err != nil {
-		return nil, fmt.Errorf("crawl: %w", err)
-	}
-
-	buf := bytes.NewBuffer(nil)
-	if err := html.Render(buf, htmlNode); err != nil {
-		return nil, fmt.Errorf("render result html: %w", err)
-	}
-
-	htmlFile := entity.NewFile("page.html", buf.Bytes())
-
-	return []entity.File{htmlFile}, nil
-}
-
-func (s *SingleFile) crawl(ctx context.Context, node *html.Node, baseURL string, encoding string) error {
-	if node.Data == "head" {
-		s.setCharset(node, encoding)
-	}
-
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		if child.Type == html.ElementNode {
-			if err := s.findAndReplaceResources(ctx, child, baseURL); err != nil {
-				return err
-			}
-		}
-
-		if err := s.crawl(ctx, child, baseURL, encoding); err != nil {
-			return fmt.Errorf("crawl child %s: %w", child.Data, err)
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) findAndReplaceResources(ctx context.Context, node *html.Node, baseURL string) error {
-	switch node.DataAtom {
-	case atom.Img, atom.Image, atom.Script, atom.Style:
-		err := s.replaceResource(ctx, node, baseURL)
-		if err != nil {
-			return err
-		}
-
-	case atom.Link:
-		for _, attribute := range node.Attr {
-			if attribute.Key == "rel" && (attribute.Val == "stylesheet") {
-				if err := s.replaceResource(ctx, node, baseURL); err != nil {
-					return err
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) replaceResource(ctx context.Context, node *html.Node, baseURL string) error {
-	for i, attribute := range node.Attr {
-		if attribute.Key == "src" || attribute.Key == "href" {
-			encoded, contentType, err := s.loadResource(ctx, attribute.Val, baseURL)
-			if err != nil {
-				return fmt.Errorf("load resource for %s: %w", node.Data, err)
-			}
-
-			if len(encoded) == 0 {
-				attribute.Val = ""
-
-			} else {
-				attribute.Val = fmt.Sprintf("data:%s;base64, %s", contentType, encoded)
-			}
-
-			node.Attr[i] = attribute
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) loadResource(ctx context.Context, val, baseURL string) ([]byte, string, error) {
-	if !strings.HasPrefix(val, "http://") && !strings.HasPrefix(val, "https://") {
-		var err error
-		val, err = url.JoinPath(baseURL, val)
-		if err != nil {
-			return nil, "", fmt.Errorf("join base path %s and url %s: %w", baseURL, val, err)
-		}
-		val, err = url.PathUnescape(val)
-		if err != nil {
-			return nil, "", fmt.Errorf("unescape path %s: %w", val, err)
-		}
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, val, nil)
-	if err != nil {
-		return nil, "", fmt.Errorf("new request: %w", err)
-	}
-
-	response, err := s.client.Do(req)
-	if err != nil {
-		return nil, "", fmt.Errorf("do request: %w", err)
-	}
-
-	defer func() {
-		if response.Body != nil {
-			_ = response.Body.Close()
-		}
-	}()
-
-	if response.StatusCode != http.StatusOK {
-		return []byte{}, "", nil
-	}
-
-	raw, err := io.ReadAll(response.Body)
-	if err != nil {
-		return nil, "", fmt.Errorf("read body: %w", err)
-	}
-
-	encoded := make([]byte, base64.StdEncoding.EncodedLen(len(raw)))
-	base64.StdEncoding.Encode(encoded, raw)
-
-	return encoded, response.Header.Get("Content-Type"), nil
-}
-
-func (s *SingleFile) setCharset(node *html.Node, encoding string) {
-	var charsetExists bool
-
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		if child.Data == "meta" {
-			for _, attribute := range child.Attr {
-				if attribute.Key == "charset" {
-					charsetExists = true
-				}
-			}
-		}
-	}
-
-	if !charsetExists {
-		node.AppendChild(&html.Node{
-			Type:     html.ElementNode,
-			DataAtom: atom.Meta,
-			Data:     "meta",
-			Attr: []html.Attribute{
-				{
-					Key: "charset",
-					Val: encoding,
-				},
-			},
-		})
-	}
-}
-
-func baseURL(val string) string {
-	parsed, err := url.Parse(val)
-	if err != nil {
-		return val
-	}
-
-	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
-}
-
-func getEncoding(response *http.Response) string {
-	_, encoding, found := strings.Cut(response.Header.Get("Content-Type"), "charset=")
-	if !found {
-		return defaultEncoding
-	}
-
-	encoding = strings.TrimSpace(encoding)
-
-	return encoding
+	return response, nil
 }

adapters/repository/badger.go

@@ -1,4 +1,4 @@
-package badger
+package repository
 
 import (
 	"errors"

adapters/repository/badger/page.go

@@ -8,6 +8,8 @@ import (
 	"github.com/dgraph-io/badger/v4"
 	"github.com/google/uuid"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/entity"
 )
 
@@ -24,7 +26,9 @@ type Page struct {
 }
 
 func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
-	page := entity.Page{ID: pageID}
+	page := entity.Page{}
+	page.ID = pageID
+
 	var file *entity.File
 
 	err := p.db.View(func(txn *badger.Txn) error {
@@ -44,9 +48,9 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil
 			return fmt.Errorf("get value: %w", err)
 		}
 
-		for i := range page.Results.Results() {
-			for j := range page.Results.Results()[i].Files {
-				ff := &page.Results.Results()[i].Files[j]
+		for i := range page.Results {
+			for j := range page.Results[i].Files {
+				ff := &page.Results[i].Files[j]
 
 				if ff.ID == fileID {
 					file = ff
@@ -66,7 +70,7 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil
 
 func (p *Page) Save(_ context.Context, page *entity.Page) error {
 	if p.db.IsClosed() {
-		return ErrDBClosed
+		return repository.ErrDBClosed
 	}
 
 	marshaled, err := marshal(page)
@@ -88,16 +92,17 @@ func (p *Page) Save(_ context.Context, page *entity.Page) error {
 }
 
 func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
-	site := entity.Page{ID: id}
+	page := entity.Page{}
+	page.ID = id
 
 	err := p.db.View(func(txn *badger.Txn) error {
-		data, err := txn.Get(p.key(&site))
+		data, err := txn.Get(p.key(&page))
 		if err != nil {
 			return fmt.Errorf("get data: %w", err)
 		}
 
 		err = data.Value(func(val []byte) error {
-			if err := unmarshal(val, &site); err != nil {
+			if err := unmarshal(val, &page); err != nil {
 				return fmt.Errorf("unmarshal data: %w", err)
 			}
 
@@ -113,7 +118,7 @@ func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
 		return nil, fmt.Errorf("view: %w", err)
 	}
 
-	return &site, nil
+	return &page, nil
 }
 
 func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
@@ -143,16 +148,7 @@ func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
 				return fmt.Errorf("get item: %w", err)
 			}
 
-			pages = append(pages, &entity.Page{
-				ID:          page.ID,
-				URL:         page.URL,
-				Description: page.Description,
-				Created:     page.Created,
-				Formats:     page.Formats,
-				Version:     page.Version,
-				Status:      page.Status,
-				Meta:        page.Meta,
-			})
+			pages = append(pages, &page)
 		}
 
 		return nil
@@ -169,8 +165,8 @@ func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
 	return pages, nil
 }
 
-func (p *Page) ListUnprocessed(ctx context.Context) ([]*entity.Page, error) {
-	pages := make([]*entity.Page, 0, 100)
+func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
+	pages := make([]entity.Page, 0, 100)
 
 	err := p.db.View(func(txn *badger.Txn) error {
 		iterator := txn.NewIterator(badger.DefaultIteratorOptions)
@@ -196,20 +192,11 @@ func (p *Page) ListUnprocessed(ctx context.Context) ([]*entity.Page, error) {
 				return fmt.Errorf("get item: %w", err)
 			}
 
-			if page.Status != entity.StatusProcessing {
-				continue
+			if page.Status == entity.StatusNew || page.Status == entity.StatusProcessing {
+				//goland:noinspection GoVetCopyLock
+				pages = append(pages, page) //nolint:govet // didn't touch the lock here
 			}
 
-			pages = append(pages, &entity.Page{
-				ID:          page.ID,
-				URL:         page.URL,
-				Description: page.Description,
-				Created:     page.Created,
-				Formats:     page.Formats,
-				Version:     page.Version,
-				Status:      page.Status,
-				Meta:        page.Meta,
-			})
 		}
 
 		return nil

adapters/repository/badger/page_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zaptest"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/entity"
 )
 
@@ -31,7 +33,7 @@ func TestSite(t *testing.T) {
 
 	log := zaptest.NewLogger(t)
 
-	db, err := NewBadger(tempDir, log.Named("db"))
+	db, err := repository.NewBadger(tempDir, log.Named("db"))
 	require.NoError(t, err)
 
 	siteRepo, err := NewPage(db)
@@ -49,12 +51,16 @@ func TestSite(t *testing.T) {
 		storedSite, err := siteRepo.Get(ctx, site.ID)
 		require.NoError(t, err)
 
-		assert.Equal(t, site, storedSite)
+		assert.Equal(t, site.ID, storedSite.ID)
+		assert.Equal(t, site.URL, storedSite.URL)
+		assert.Equal(t, site.Status, storedSite.Status)
 
 		all, err := siteRepo.ListAll(ctx)
 		require.NoError(t, err)
 		require.Len(t, all, 1)
 
-		assert.Equal(t, site, all[0])
+		assert.Equal(t, site.ID, all[0].ID)
+		assert.Equal(t, site.URL, all[0].URL)
+		assert.Equal(t, site.Status, all[0].Status)
 	})
 }

adapters/repository/badgers3/marshal.go

@@ -0,0 +1,13 @@
+package badgers3
+
+import (
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+func marshal(v interface{}) ([]byte, error) {
+	return msgpack.Marshal(v)
+}
+
+func unmarshal(b []byte, v interface{}) error { //nolint:unused // will use later
+	return msgpack.Unmarshal(b, v)
+}

adapters/repository/badgers3/page.go

@@ -0,0 +1,119 @@
+package badgers3
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/derfenix/webarchive/adapters/repository"
+	"github.com/derfenix/webarchive/entity"
+	"github.com/dgraph-io/badger/v4"
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+)
+
+func NewPage(db *badger.DB, s3 *minio.Client, bucketName string) (*Page, error) {
+	return &Page{
+		db:         db,
+		s3:         s3,
+		prefix:     []byte("pages3:"),
+		bucketName: bucketName,
+	}, nil
+}
+
+type Page struct {
+	db         *badger.DB
+	s3         *minio.Client
+	prefix     []byte
+	bucketName string
+}
+
+func (p *Page) ListAll(ctx context.Context) ([]*entity.PageBase, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) Get(ctx context.Context, id uuid.UUID) (*entity.Page, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) GetFile(ctx context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) Save(ctx context.Context, page *entity.Page) error {
+	if p.db.IsClosed() {
+		return repository.ErrDBClosed
+	}
+
+	marshaled, err := marshal(page.PageBase)
+	if err != nil {
+		return fmt.Errorf("marshal data: %w", err)
+	}
+
+	if err := p.db.Update(func(txn *badger.Txn) error {
+		if err := txn.Set(p.key(page), marshaled); err != nil {
+			return fmt.Errorf("put data: %w", err)
+		}
+
+		return nil
+	}); err != nil {
+		return fmt.Errorf("update db: %w", err)
+	}
+
+	snowball := make(chan minio.SnowballObject, 1)
+
+	go func() {
+		defer close(snowball)
+
+		for _, result := range page.Results {
+			for _, file := range result.Files {
+				for {
+					if ctx.Err() != nil {
+						return
+					}
+
+					if len(snowball) < cap(snowball) {
+						break
+					}
+				}
+
+				snowball <- minio.SnowballObject{
+					Key:     file.ID.String(),
+					Size:    int64(len(file.Data)),
+					ModTime: time.Now(),
+					Content: bytes.NewReader(file.Data),
+				}
+			}
+		}
+	}()
+
+	if err = p.s3.PutObjectsSnowball(ctx, p.bucketName, minio.SnowballOptions{Compress: true}, snowball); err != nil {
+		if dErr := p.db.Update(func(txn *badger.Txn) error {
+			if err := txn.Delete(p.key(page)); err != nil {
+				return fmt.Errorf("put data: %w", err)
+			}
+
+			return nil
+		}); dErr != nil {
+			err = errors.Join(err, dErr)
+		}
+
+		return fmt.Errorf("store files to s3: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) key(site *entity.Page) []byte {
+	return append(p.prefix, []byte(site.ID.String())...)
+}

api/gen.go

@@ -1,3 +1,3 @@
 package api
 
-//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.60.1 --target ./openapi -package openapi --clean openapi.yaml
+//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.77.0 --target ./openapi -package openapi --clean openapi.yaml

api/openapi.yaml

@@ -1,4 +1,5 @@
-openapi: 3.1.0
+---
+openapi: 3.0.3
 info:
   title: Sample API
   description: API description in Markdown.
@@ -125,7 +126,7 @@ paths:
         200:
           description: File content
           content:
-            application/pdf: { }
+            application/pdf: {}
             text/plain:
               schema:
                 type: string

api/openapi/oas_cfg_gen.go

@@ -7,7 +7,6 @@ import (
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/trace"
 
 	ht "github.com/ogen-go/ogen/http"
@@ -40,7 +39,7 @@ func (cfg *otelConfig) initOTEL() {
 		cfg.TracerProvider = otel.GetTracerProvider()
 	}
 	if cfg.MeterProvider == nil {
-		cfg.MeterProvider = metric.NewNoopMeterProvider()
+		cfg.MeterProvider = otel.GetMeterProvider()
 	}
 	cfg.Tracer = cfg.TracerProvider.Tracer(otelogen.Name,
 		trace.WithInstrumentationVersion(otelogen.SemVersion()),
@@ -99,9 +98,9 @@ func newServerConfig(opts ...ServerOption) serverConfig {
 
 type baseServer struct {
 	cfg      serverConfig
-	requests instrument.Int64Counter
-	errors   instrument.Int64Counter
-	duration instrument.Int64Histogram
+	requests metric.Int64Counter
+	errors   metric.Int64Counter
+	duration metric.Float64Histogram
 }
 
 func (s baseServer) notFound(w http.ResponseWriter, r *http.Request) {
@@ -120,7 +119,7 @@ func (cfg serverConfig) baseServer() (s baseServer, err error) {
 	if s.errors, err = s.cfg.Meter.Int64Counter(otelogen.ServerErrorsCount); err != nil {
 		return s, err
 	}
-	if s.duration, err = s.cfg.Meter.Int64Histogram(otelogen.ServerDuration); err != nil {
+	if s.duration, err = s.cfg.Meter.Float64Histogram(otelogen.ServerDuration); err != nil {
 		return s, err
 	}
 	return s, nil
@@ -162,9 +161,9 @@ func newClientConfig(opts ...ClientOption) clientConfig {
 
 type baseClient struct {
 	cfg      clientConfig
-	requests instrument.Int64Counter
-	errors   instrument.Int64Counter
-	duration instrument.Int64Histogram
+	requests metric.Int64Counter
+	errors   metric.Int64Counter
+	duration metric.Float64Histogram
 }
 
 func (cfg clientConfig) baseClient() (c baseClient, err error) {
@@ -175,7 +174,7 @@ func (cfg clientConfig) baseClient() (c baseClient, err error) {
 	if c.errors, err = c.cfg.Meter.Int64Counter(otelogen.ClientErrorsCount); err != nil {
 		return c, err
 	}
-	if c.duration, err = c.cfg.Meter.Int64Histogram(otelogen.ClientDuration); err != nil {
+	if c.duration, err = c.cfg.Meter.Float64Histogram(otelogen.ClientDuration); err != nil {
 		return c, err
 	}
 	return c, nil
@@ -200,7 +199,7 @@ func WithTracerProvider(provider trace.TracerProvider) Option {
 
 // WithMeterProvider specifies a meter provider to use for creating a meter.
 //
-// If none is specified, the metric.NewNoopMeterProvider is used.
+// If none is specified, the otel.GetMeterProvider() is used.
 func WithMeterProvider(provider metric.MeterProvider) Option {
 	return otelOptionFunc(func(cfg *otelConfig) {
 		if provider != nil {

api/openapi/oas_client_gen.go

@@ -11,6 +11,8 @@ import (
 	"github.com/go-faster/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/ogen-go/ogen/conv"
@@ -19,6 +21,34 @@ import (
 	"github.com/ogen-go/ogen/uri"
 )
 
+// Invoker invokes operations described by OpenAPI v3 specification.
+type Invoker interface {
+	// AddPage invokes addPage operation.
+	//
+	// Add new page.
+	//
+	// POST /pages
+	AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error)
+	// GetFile invokes getFile operation.
+	//
+	// Get file content.
+	//
+	// GET /pages/{id}/file/{file_id}
+	GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error)
+	// GetPage invokes getPage operation.
+	//
+	// Get page details.
+	//
+	// GET /pages/{id}
+	GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error)
+	// GetPages invokes getPages operation.
+	//
+	// Get all pages.
+	//
+	// GET /pages
+	GetPages(ctx context.Context) (Pages, error)
+}
+
 // Client implements OAS client.
 type Client struct {
 	serverURL *url.URL
@@ -78,19 +108,20 @@ func (c *Client) requestURL(ctx context.Context) *url.URL {
 // POST /pages
 func (c *Client) AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error) {
 	res, err := c.sendAddPage(ctx, request, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (res AddPageRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("addPage"),
+		semconv.HTTPMethodKey.String("POST"),
+		semconv.HTTPRouteKey.String("/pages"),
 	}
 	// Validate request before sending.
 	if err := func() error {
-		if request.Set {
+		if value, ok := request.Get(); ok {
 			if err := func() error {
-				if err := request.Value.Validate(); err != nil {
+				if err := value.Validate(); err != nil {
 					return err
 				}
 				return nil
@@ -106,12 +137,13 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "AddPage",
@@ -124,7 +156,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -197,7 +229,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 	u.RawQuery = q.Values().Encode()
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "POST", u, nil)
+	r, err := ht.NewRequest(ctx, "POST", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -228,24 +260,26 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 // GET /pages/{id}/file/{file_id}
 func (c *Client) GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error) {
 	res, err := c.sendGetFile(ctx, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res GetFileRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getFile"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages/{id}/file/{file_id}"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetFile",
@@ -258,7 +292,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -307,7 +341,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -335,24 +369,26 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 // GET /pages/{id}
 func (c *Client) GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error) {
 	res, err := c.sendGetPage(ctx, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res GetPageRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getPage"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages/{id}"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetPage",
@@ -365,7 +401,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -395,7 +431,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -423,24 +459,26 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 // GET /pages
 func (c *Client) GetPages(ctx context.Context) (Pages, error) {
 	res, err := c.sendGetPages(ctx)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getPages"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetPages",
@@ -453,7 +491,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -465,7 +503,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}

api/openapi/oas_handlers_gen.go

@@ -10,7 +10,8 @@ import (
 	"github.com/go-faster/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
-	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
 	"go.opentelemetry.io/otel/trace"
 
 	ht "github.com/ogen-go/ogen/http"
@@ -42,17 +43,18 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -89,10 +91,11 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 	var response AddPageRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "AddPage",
-			OperationID:   "addPage",
-			Body:          request,
+			Context:          ctx,
+			OperationName:    "AddPage",
+			OperationSummary: "Add new page",
+			OperationID:      "addPage",
+			Body:             request,
 			Params: middleware.Parameters{
 				{
 					Name: "url",
@@ -132,22 +135,27 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 		response, err = s.h.AddPage(ctx, request, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeAddPageResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -175,17 +183,18 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -207,10 +216,11 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 	var response GetFileRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "GetFile",
-			OperationID:   "getFile",
-			Body:          nil,
+			Context:          ctx,
+			OperationName:    "GetFile",
+			OperationSummary: "",
+			OperationID:      "getFile",
+			Body:             nil,
 			Params: middleware.Parameters{
 				{
 					Name: "id",
@@ -246,22 +256,27 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 		response, err = s.h.GetFile(ctx, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetFileResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -289,17 +304,18 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -321,10 +337,11 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 	var response GetPageRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "GetPage",
-			OperationID:   "getPage",
-			Body:          nil,
+			Context:          ctx,
+			OperationName:    "GetPage",
+			OperationSummary: "",
+			OperationID:      "getPage",
+			Body:             nil,
 			Params: middleware.Parameters{
 				{
 					Name: "id",
@@ -356,22 +373,27 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 		response, err = s.h.GetPage(ctx, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetPageResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -399,17 +421,18 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err error
 	)
@@ -417,12 +440,13 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 	var response Pages
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
-			OperationName: "GetPages",
-			OperationID:   "getPages",
-			Body:          nil,
-			Params:        middleware.Parameters{},
-			Raw:           r,
+			Context:          ctx,
+			OperationName:    "GetPages",
+			OperationSummary: "Get all pages",
+			OperationID:      "getPages",
+			Body:             nil,
+			Params:           middleware.Parameters{},
+			Raw:              r,
 		}
 
 		type (
@@ -447,22 +471,27 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 		response, err = s.h.GetPages(ctx)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetPagesResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }

api/openapi/oas_json_gen.go

@@ -23,12 +23,10 @@ func (s *AddPageBadRequest) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *AddPageBadRequest) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("field")
 		e.Str(s.Field)
 	}
 	{
-
 		e.FieldStart("error")
 		e.Str(s.Error)
 	}
@@ -138,7 +136,6 @@ func (s *AddPageReq) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *AddPageReq) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
@@ -280,7 +277,6 @@ func (s *Error) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Error) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("message")
 		e.Str(s.Message)
 	}
@@ -506,22 +502,18 @@ func (s *Page) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Page) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
 	{
-
 		e.FieldStart("created")
 		json.EncodeDateTime(e, s.Created)
 	}
 	{
-
 		e.FieldStart("formats")
 		e.ArrStart()
 		for _, elem := range s.Formats {
@@ -530,12 +522,10 @@ func (s *Page) encodeFields(e *jx.Encoder) {
 		e.ArrEnd()
 	}
 	{
-
 		e.FieldStart("status")
 		s.Status.Encode(e)
 	}
 	{
-
 		e.FieldStart("meta")
 		s.Meta.Encode(e)
 	}
@@ -699,12 +689,10 @@ func (s *PageMeta) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *PageMeta) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("title")
 		e.Str(s.Title)
 	}
 	{
-
 		e.FieldStart("description")
 		e.Str(s.Description)
 	}
@@ -831,22 +819,18 @@ func (s *PageWithResults) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *PageWithResults) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
 	{
-
 		e.FieldStart("created")
 		json.EncodeDateTime(e, s.Created)
 	}
 	{
-
 		e.FieldStart("formats")
 		e.ArrStart()
 		for _, elem := range s.Formats {
@@ -855,17 +839,14 @@ func (s *PageWithResults) encodeFields(e *jx.Encoder) {
 		e.ArrEnd()
 	}
 	{
-
 		e.FieldStart("status")
 		s.Status.Encode(e)
 	}
 	{
-
 		e.FieldStart("meta")
 		s.Meta.Encode(e)
 	}
 	{
-
 		e.FieldStart("results")
 		e.ArrStart()
 		for _, elem := range s.Results {
@@ -1052,12 +1033,10 @@ func (s *PageWithResultsMeta) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *PageWithResultsMeta) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("title")
 		e.Str(s.Title)
 	}
 	{
-
 		e.FieldStart("description")
 		e.Str(s.Description)
 	}
@@ -1234,7 +1213,6 @@ func (s *Result) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Result) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("format")
 		s.Format.Encode(e)
 	}
@@ -1245,7 +1223,6 @@ func (s *Result) encodeFields(e *jx.Encoder) {
 		}
 	}
 	{
-
 		e.FieldStart("files")
 		e.ArrStart()
 		for _, elem := range s.Files {
@@ -1374,22 +1351,18 @@ func (s *ResultFilesItem) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *ResultFilesItem) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("name")
 		e.Str(s.Name)
 	}
 	{
-
 		e.FieldStart("mimetype")
 		e.Str(s.Mimetype)
 	}
 	{
-
 		e.FieldStart("size")
 		e.Int64(s.Size)
 	}

api/openapi/oas_request_decoders_gen.go

@@ -77,9 +77,9 @@ func (s *Server) decodeAddPageRequest(r *http.Request) (
 			return req, close, err
 		}
 		if err := func() error {
-			if request.Set {
+			if value, ok := request.Get(); ok {
 				if err := func() error {
-					if err := request.Value.Validate(); err != nil {
+					if err := value.Validate(); err != nil {
 						return err
 					}
 					return nil

api/openapi/oas_request_encoders_gen.go

@@ -20,7 +20,7 @@ func encodeAddPageRequest(
 		// Keep request with empty body if value is not set.
 		return nil
 	}
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	{
 		if req.Set {
 			req.Encode(e)

api/openapi/oas_response_decoders_gen.go

@@ -15,7 +15,7 @@ import (
 	"github.com/ogen-go/ogen/validate"
 )
 
-func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
+func decodeAddPageResponse(resp *http.Response) (res AddPageRes, _ error) {
 	switch resp.StatusCode {
 	case 201:
 		// Code 201.
@@ -128,12 +128,12 @@ func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
+func decodeGetFileResponse(resp *http.Response) (res GetFileRes, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -216,12 +216,12 @@ func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
+func decodeGetPageResponse(resp *http.Response) (res GetPageRes, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -302,12 +302,12 @@ func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
+func decodeGetPagesResponse(resp *http.Response) (res Pages, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -385,7 +385,7 @@ func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }

api/openapi/oas_response_encoders_gen.go

@@ -10,6 +10,8 @@ import (
 	"github.com/go-faster/jx"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
+
+	ht "github.com/ogen-go/ogen/http"
 )
 
 func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trace.Span) error {
@@ -19,11 +21,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(201)
 		span.SetStatus(codes.Ok, http.StatusText(201))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *AddPageBadRequest:
@@ -31,11 +34,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(400)
 		span.SetStatus(codes.Error, http.StatusText(400))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	default:
@@ -54,6 +58,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileOKTextHTML:
@@ -65,6 +70,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileOKTextPlain:
@@ -76,6 +82,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileNotFound:
@@ -96,11 +103,12 @@ func encodeGetPageResponse(response GetPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(200)
 		span.SetStatus(codes.Ok, http.StatusText(200))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetPageNotFound:
@@ -119,11 +127,12 @@ func encodeGetPagesResponse(response Pages, w http.ResponseWriter, span trace.Sp
 	w.WriteHeader(200)
 	span.SetStatus(codes.Ok, http.StatusText(200))
 
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	response.Encode(e)
 	if _, err := e.WriteTo(w); err != nil {
 		return errors.Wrap(err, "write")
 	}
+
 	return nil
 }
 
@@ -142,11 +151,15 @@ func encodeErrorResponse(response *ErrorStatusCode, w http.ResponseWriter, span
 		span.SetStatus(codes.Ok, st)
 	}
 
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	response.Response.Encode(e)
 	if _, err := e.WriteTo(w); err != nil {
 		return errors.Wrap(err, "write")
 	}
+
+	if code >= http.StatusInternalServerError {
+		return errors.Wrapf(ht.ErrInternalServerErrorResponse, "code: %d, message: %s", code, http.StatusText(code))
+	}
 	return nil
 
 }

api/openapi/oas_router_gen.go

@@ -10,6 +10,19 @@ import (
 	"github.com/ogen-go/ogen/uri"
 )
 
+func (s *Server) cutPrefix(path string) (string, bool) {
+	prefix := s.cfg.Prefix
+	if prefix == "" {
+		return path, true
+	}
+	if !strings.HasPrefix(path, prefix) {
+		// Prefix doesn't match.
+		return "", false
+	}
+	// Cut prefix from the path.
+	return strings.TrimPrefix(path, prefix), true
+}
+
 // ServeHTTP serves http request as defined by OpenAPI v3 specification,
 // calling handler that matches the path or returning not found error.
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -21,17 +34,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			elemIsEscaped = strings.ContainsRune(elem, '%')
 		}
 	}
-	if prefix := s.cfg.Prefix; len(prefix) > 0 {
-		if strings.HasPrefix(elem, prefix) {
-			// Cut prefix from the path.
-			elem = strings.TrimPrefix(elem, prefix)
-		} else {
-			// Prefix doesn't match.
-			s.notFound(w, r)
-			return
-		}
-	}
-	if len(elem) == 0 {
+
+	elem, ok := s.cutPrefix(elem)
+	if !ok || len(elem) == 0 {
 		s.notFound(w, r)
 		return
 	}
@@ -129,6 +134,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 // Route is route object.
 type Route struct {
 	name        string
+	summary     string
 	operationID string
 	pathPattern string
 	count       int
@@ -142,6 +148,11 @@ func (r Route) Name() string {
 	return r.name
 }
 
+// Summary returns OpenAPI summary.
+func (r Route) Summary() string {
+	return r.summary
+}
+
 // OperationID returns OpenAPI operationId.
 func (r Route) OperationID() string {
 	return r.operationID
@@ -183,6 +194,11 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 		}()
 	}
 
+	elem, ok := s.cutPrefix(elem)
+	if !ok {
+		return r, false
+	}
+
 	// Static code generated router with unwrapped path search.
 	switch {
 	default:
@@ -201,6 +217,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 				switch method {
 				case "GET":
 					r.name = "GetPages"
+					r.summary = "Get all pages"
 					r.operationID = "getPages"
 					r.pathPattern = "/pages"
 					r.args = args
@@ -208,6 +225,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 					return r, true
 				case "POST":
 					r.name = "AddPage"
+					r.summary = "Add new page"
 					r.operationID = "addPage"
 					r.pathPattern = "/pages"
 					r.args = args
@@ -238,6 +256,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 					switch method {
 					case "GET":
 						r.name = "GetPage"
+						r.summary = ""
 						r.operationID = "getPage"
 						r.pathPattern = "/pages/{id}"
 						r.args = args
@@ -265,6 +284,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 						case "GET":
 							// Leaf: GetFile
 							r.name = "GetFile"
+							r.summary = ""
 							r.operationID = "getFile"
 							r.pathPattern = "/pages/{id}/file/{file_id}"
 							r.args = args

api/openapi/oas_schemas_gen.go

@@ -140,6 +140,16 @@ const (
 	FormatHeaders    Format = "headers"
 )
 
+// AllValues returns all Format values.
+func (Format) AllValues() []Format {
+	return []Format{
+		FormatAll,
+		FormatPdf,
+		FormatSingleFile,
+		FormatHeaders,
+	}
+}
+
 // MarshalText implements encoding.TextMarshaler.
 func (s Format) MarshalText() ([]byte, error) {
 	switch s {
@@ -189,6 +199,9 @@ type GetFileOKApplicationPdf struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKApplicationPdf) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -202,6 +215,9 @@ type GetFileOKTextHTML struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKTextHTML) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -215,6 +231,9 @@ type GetFileOKTextPlain struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKTextPlain) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -642,6 +661,17 @@ const (
 	StatusWithErrors Status = "with_errors"
 )
 
+// AllValues returns all Status values.
+func (Status) AllValues() []Status {
+	return []Status{
+		StatusNew,
+		StatusProcessing,
+		StatusDone,
+		StatusFailed,
+		StatusWithErrors,
+	}
+}
+
 // MarshalText implements encoding.TextMarshaler.
 func (s Status) MarshalText() ([]byte, error) {
 	switch s {

api/openapi/oas_validators_gen.go

@@ -42,6 +42,7 @@ func (s *AddPageReq) Validate() error {
 	}
 	return nil
 }
+
 func (s Format) Validate() error {
 	switch s {
 	case "all":
@@ -103,6 +104,7 @@ func (s *Page) Validate() error {
 	}
 	return nil
 }
+
 func (s *PageWithResults) Validate() error {
 	var failures []validate.FieldError
 	if err := func() error {
@@ -177,12 +179,14 @@ func (s *PageWithResults) Validate() error {
 	}
 	return nil
 }
+
 func (s Pages) Validate() error {
-	if s == nil {
+	alias := ([]Page)(s)
+	if alias == nil {
 		return errors.New("nil is invalid value")
 	}
 	var failures []validate.FieldError
-	for i, elem := range s {
+	for i, elem := range alias {
 		if err := func() error {
 			if err := elem.Validate(); err != nil {
 				return err
@@ -200,6 +204,7 @@ func (s Pages) Validate() error {
 	}
 	return nil
 }
+
 func (s *Result) Validate() error {
 	var failures []validate.FieldError
 	if err := func() error {
@@ -229,6 +234,7 @@ func (s *Result) Validate() error {
 	}
 	return nil
 }
+
 func (s Status) Validate() error {
 	switch s {
 	case "new":

application/application.go

@@ -16,6 +16,8 @@ import (
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/adapters/processors"
 	badgerRepo "github.com/derfenix/webarchive/adapters/repository/badger"
 	"github.com/derfenix/webarchive/api/openapi"
@@ -30,7 +32,7 @@ func NewApplication(cfg config.Config) (Application, error) {
 		return Application{}, fmt.Errorf("new logger: %w", err)
 	}
 
-	db, err := badgerRepo.NewBadger(cfg.DB.Path, log.Named("db"))
+	db, err := repository.NewBadger(cfg.DB.Path, log.Named("db"))
 	if err != nil {
 		return Application{}, fmt.Errorf("new badger: %w", err)
 	}
@@ -40,7 +42,7 @@ func NewApplication(cfg config.Config) (Application, error) {
 		return Application{}, fmt.Errorf("new page repo: %w", err)
 	}
 
-	processor, err := processors.NewProcessors(cfg)
+	processor, err := processors.NewProcessors(cfg, log.Named("processor"))
 	if err != nil {
 		return Application{}, fmt.Errorf("new processors: %w", err)
 	}
@@ -173,7 +175,7 @@ func (a *Application) Stop() error {
 		errs = multierr.Append(errs, fmt.Errorf("sync db: %w", err))
 	}
 
-	if err := badgerRepo.Backup(a.db, badgerRepo.BackupStop); err != nil {
+	if err := repository.Backup(a.db, repository.BackupStop); err != nil {
 		errs = multierr.Append(errs, fmt.Errorf("backup on stop: %w", err))
 	}
 
@@ -189,6 +191,7 @@ func newLogger(cfg config.Logging) (*zap.Logger, error) {
 	logCfg.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder
 	logCfg.EncoderConfig.EncodeDuration = zapcore.NanosDurationEncoder
 	logCfg.DisableCaller = true
+	logCfg.DisableStacktrace = true
 
 	logCfg.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
 	if cfg.Debug {

entity/cache.go

@@ -0,0 +1,42 @@
+package entity
+
+import (
+	"bytes"
+	"io"
+	"sync"
+)
+
+func NewCache() *Cache {
+	return &Cache{data: make([]byte, 0, 1024*512)}
+}
+
+type Cache struct {
+	mu   sync.RWMutex
+	data []byte
+}
+
+func (c *Cache) Write(p []byte) (n int, err error) {
+	c.mu.Lock()
+	c.data = append(c.data, p...)
+	c.mu.Unlock()
+
+	return len(p), nil
+}
+
+func (c *Cache) Get() []byte {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.data
+}
+
+func (c *Cache) Reader() io.Reader {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if len(c.data) == 0 {
+		return nil
+	}
+
+	return bytes.NewBuffer(c.data)
+}

entity/page.go

@@ -3,6 +3,7 @@ package entity
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -10,8 +11,8 @@ import (
 )
 
 type Processor interface {
-	Process(ctx context.Context, format Format, url string) Result
-	GetMeta(ctx context.Context, url string) (Meta, error)
+	Process(ctx context.Context, format Format, url string, cache *Cache) Result
+	GetMeta(ctx context.Context, url string, cache *Cache) (Meta, error)
 }
 
 type Format uint8
@@ -41,59 +42,79 @@ const (
 type Meta struct {
 	Title       string
 	Description string
+	Encoding    string
 	Error       string
 }
 
-func NewPage(url string, description string, formats ...Format) *Page {
-	return &Page{
-		ID:          uuid.New(),
-		URL:         url,
-		Description: description,
-		Formats:     formats,
-		Created:     time.Now(),
-		Version:     1,
-	}
-}
-
-type Page struct {
+type PageBase struct {
 	ID          uuid.UUID
 	URL         string
 	Description string
 	Created     time.Time
 	Formats     []Format
-	Results     Results
 	Version     uint16
 	Status      Status
 	Meta        Meta
 }
 
+func NewPage(url string, description string, formats ...Format) *Page {
+	return &Page{
+		PageBase: PageBase{
+			ID:          uuid.New(),
+			URL:         url,
+			Description: description,
+			Formats:     formats,
+			Created:     time.Now(),
+			Version:     1,
+		},
+		cache: NewCache(),
+	}
+}
+
+type Page struct {
+	PageBase
+	Results ResultsRO
+	cache   *Cache
+}
+
 func (p *Page) SetProcessing() {
 	p.Status = StatusProcessing
 }
 
+func (p *Page) Prepare(ctx context.Context, processor Processor) {
+	meta, err := processor.GetMeta(ctx, p.URL, p.cache)
+	if err != nil {
+		p.Meta.Error = err.Error()
+	} else {
+		p.Meta = meta
+	}
+}
+
 func (p *Page) Process(ctx context.Context, processor Processor) {
 	innerWG := sync.WaitGroup{}
 	innerWG.Add(len(p.Formats))
 
+	results := Results{}
+
 	for _, format := range p.Formats {
 		go func(format Format) {
 			defer innerWG.Done()
 
 			defer func() {
 				if err := recover(); err != nil {
-					p.Results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v", err)})
+					results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v (%s)", err, string(debug.Stack()))})
 				}
 			}()
 
-			result := processor.Process(ctx, format, p.URL)
-			p.Results.Add(result)
+			result := processor.Process(ctx, format, p.URL, p.cache)
+			results.Add(result)
 		}(format)
 	}
 
 	innerWG.Wait()
 
 	var hasResultWithOutErrors bool
-	for _, result := range p.Results.Results() {
+	for _, result := range results.Results() {
 		if result.Err != nil {
 			p.Status = StatusWithErrors
 		} else {
@@ -108,4 +129,6 @@ func (p *Page) Process(ctx context.Context, processor Processor) {
 	if p.Status == StatusProcessing {
 		p.Status = StatusDone
 	}
+
+	p.Results = results.RO()
 }

entity/result.go

@@ -6,6 +6,8 @@ import (
 	"github.com/vmihailenco/msgpack/v5"
 )
 
+type ResultsRO []Result
+
 type Result struct {
 	Format Format
 	Err    error
@@ -17,6 +19,13 @@ type Results struct {
 	results []Result
 }
 
+func (r *Results) RO() ResultsRO {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.results
+}
+
 func (r *Results) MarshalMsgpack() ([]byte, error) {
 	return msgpack.Marshal(r.results)
 }

entity/worker.go

@@ -9,7 +9,7 @@ import (
 
 type Pages interface {
 	Save(ctx context.Context, page *Page) error
-	ListUnprocessed(ctx context.Context) ([]*Page, error)
+	ListUnprocessed(ctx context.Context) ([]Page, error)
 }
 
 func NewWorker(ch chan *Page, pages Pages, processor Processor, log *zap.Logger) *Worker {
@@ -37,7 +37,7 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
 			w.log.Error("failed to get unprocessed pages", zap.Error(err))
 		} else {
 			for i := range unprocessed {
-				w.ch <- unprocessed[i]
+				w.ch <- &unprocessed[i]
 			}
 		}
 	}()
@@ -66,6 +66,16 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
 func (w *Worker) do(ctx context.Context, wg *sync.WaitGroup, page *Page, log *zap.Logger) {
 	defer wg.Done()
 
+	page.SetProcessing()
+	if err := w.pages.Save(ctx, page); err != nil {
+		w.log.Error(
+			"failed to save processing page",
+			zap.String("page_id", page.ID.String()),
+			zap.String("page_url", page.URL),
+			zap.Error(err),
+		)
+	}
+
 	page.Process(ctx, w.processor)
 
 	log.Debug("page processed")

go.mod

@@ -5,33 +5,35 @@ go 1.19
 require (
 	github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0
 	github.com/dgraph-io/badger/v4 v4.0.1
+	github.com/disintegration/imaging v1.6.2
 	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/go-faster/errors v0.6.1
-	github.com/go-faster/jx v1.0.0
-	github.com/google/uuid v1.3.0
-	github.com/ogen-go/ogen v0.60.1
+	github.com/go-faster/jx v1.1.0
+	github.com/google/uuid v1.4.0
+	github.com/minio/minio-go/v7 v7.0.52
+	github.com/ogen-go/ogen v0.77.0
 	github.com/sethvargo/go-envconfig v0.9.0
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
 	github.com/vmihailenco/msgpack/v5 v5.3.5
-	go.opentelemetry.io/otel v1.14.0
-	go.opentelemetry.io/otel/metric v0.37.0
-	go.opentelemetry.io/otel/trace v1.14.0
-	go.uber.org/multierr v1.10.0
-	go.uber.org/zap v1.24.0
+	go.opentelemetry.io/otel v1.19.0
+	go.opentelemetry.io/otel/metric v1.19.0
+	go.opentelemetry.io/otel/trace v1.19.0
+	go.uber.org/multierr v1.11.0
+	go.uber.org/zap v1.26.0
+	golang.org/x/net v0.23.0
 )
 
 require (
-	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
-	github.com/dlclark/regexp2 v1.8.1 // indirect
+	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-faster/yamlx v0.4.1 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-faster/yaml v0.4.6 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.1.1 // indirect
@@ -39,21 +41,30 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v23.3.3+incompatible // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rs/xid v1.4.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
-	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
+	golang.org/x/image v0.18.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -4,8 +4,6 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0 h1:DNrExYwvyyI404SxdUCCANAj9TwnGjRfa3cYFMNY1AU=
 github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0/go.mod h1:SQq4xfIdvf6WYKSDxAJc+xOJdolt+/bc1jnQKMtPMvQ=
-github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
-github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -23,8 +21,10 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa
 github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0=
-github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -40,13 +40,13 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI=
 github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY=
-github.com/go-faster/jx v1.0.0 h1:HE+ms2e6ZGkZ6u13t8u+onBinrPvIPI+0hWXGELm74g=
-github.com/go-faster/jx v1.0.0/go.mod h1:zm8SlkwK+H0TYNKYtVJ/7cWFS7soJBQWhcPctKyYL/4=
-github.com/go-faster/yamlx v0.4.1 h1:00RQkZopoLDF1SgBDJVHuN6epTOK7T0TkN427vbvEBk=
-github.com/go-faster/yamlx v0.4.1/go.mod h1:QXr/i3Z00jRhskgyWkoGsEdseebd/ZbZEpGS6DJv8oo=
+github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg=
+github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg=
+github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
+github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -82,32 +82,54 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
 github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
+github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/ogen-go/ogen v0.60.1 h1:yOt0i6NcH7jM3rBi9nnv5VsGUQRw4ACUMsiJojnqrAM=
-github.com/ogen-go/ogen v0.60.1/go.mod h1:tcwLpHe4vyk9xtbTMe3yu3Qtcbz8VjrpBz9LzsdwWvQ=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps=
+github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/ogen-go/ogen v0.77.0 h1:yREPDg3cDuXkDyp7FPXdPEUz+azPZFUGKmYer8fJpmM=
+github.com/ogen-go/ogen v0.77.0/go.mod h1:/bl+MubIppovr7F1fKAaDxzFF+oF2EiMtyVylyqDtQ8=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE=
 github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -116,11 +138,12 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
 github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
@@ -129,25 +152,28 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
-go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
-go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
-go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
-go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
-go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
-go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
+go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
+golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -162,28 +188,31 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -217,10 +246,12 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

ports/rest/converter.go

@@ -29,10 +29,10 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults {
 			Error:       openapi.NewOptString(page.Meta.Error),
 		},
 		Results: func() []openapi.Result {
-			results := make([]openapi.Result, len(page.Results.Results()))
+			results := make([]openapi.Result, len(page.Results))
 
 			for i := range results {
-				result := &(page.Results.Results())[i]
+				result := &page.Results[i]
 
 				errText := openapi.OptString{}
 				if result.Err != nil {
@@ -66,6 +66,29 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults {
 	}
 }
 
+func BasePageToRest(page *entity.PageBase) openapi.Page {
+	return openapi.Page{
+		ID:      page.ID,
+		URL:     page.URL,
+		Created: page.Created,
+		Meta: openapi.PageMeta{
+			Title:       html.EscapeString(page.Meta.Title),
+			Description: html.EscapeString(page.Meta.Description),
+			Error:       openapi.NewOptString(page.Meta.Error),
+		},
+		Formats: func() []openapi.Format {
+			res := make([]openapi.Format, len(page.Formats))
+
+			for i, format := range page.Formats {
+				res[i] = FormatToRest(format)
+			}
+
+			return res
+		}(),
+		Status: StatusToRest(page.Status),
+	}
+}
+
 func PageToRest(page *entity.Page) openapi.Page {
 	return openapi.Page{
 		ID:      page.ID,

ports/rest/service.go

@@ -30,9 +30,9 @@ func NewService(pages Pages, ch chan *entity.Page, processor entity.Processor) *
 
 type Service struct {
 	openapi.UnimplementedHandler
+	processor entity.Processor
 	pages     Pages
 	ch        chan *entity.Page
-	processor entity.Processor
 }
 
 func (s *Service) GetPage(ctx context.Context, params openapi.GetPageParams) (openapi.GetPageRes, error) {
@@ -81,20 +81,14 @@ func (s *Service) AddPage(ctx context.Context, req openapi.OptAddPageReq, params
 	}
 
 	page := entity.NewPage(url, description, domainFormats...)
-	page.Status = entity.StatusProcessing
-
-	meta, err := s.processor.GetMeta(ctx, page.URL)
-	if err != nil {
-		page.Meta.Error = err.Error()
-	} else {
-		page.Meta = meta
-	}
+	page.Status = entity.StatusNew
+	page.Prepare(ctx, s.processor)
 
 	if err := s.pages.Save(ctx, page); err != nil {
 		return nil, fmt.Errorf("save page: %w", err)
 	}
 
-	res := PageToRest(page)
+	res := BasePageToRest(&page.PageBase)
 
 	s.ch <- page