Merge pull request #11 from derfenix/dependabot/go_modules/google.golang.org/protobuf-1.33.0

Bump google.golang.org/protobuf from 1.30.0 to 1.33.0

.github/workflows/release.yaml

@@ -1,6 +1,7 @@
+---
 name: release
 
-on:
+"on":
   push:
     tags:
       - 'v*'

.github/workflows/test.yaml

@@ -1,6 +1,7 @@
+---
 name: test
 
-on:
+"on":
   pull_request:
   push:
     branches:

.gitignore

@@ -43,3 +43,5 @@ fabric.properties
 go.work
 test.http
 db
+http-client.env.json
+http-client.private.env.json

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

.idea/jsonSchemas.xml

@@ -3,11 +3,11 @@
   <component name="JsonSchemaMappingsProjectConfiguration">
     <state>
       <map>
-        <entry key="openapi">
+        <entry key="OpenAPI 3.0">
           <value>
             <SchemaInfo>
-              <option name="name" value="openapi" />
+              <option name="name" value="OpenAPI 3.0" />
-              <option name="relativePathToSchema" value="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json" />
+              <option name="relativePathToSchema" value="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/schemas/v3.0/schema.json" />
               <option name="applicationDefined" value="true" />
               <option name="patterns">
                 <list>

.idea/swagger-settings.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SwaggerSettings">
+    <option name="defaultPreviewType" value="SWAGGER_UI" />
+  </component>
+</project>

.idea/webResources.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="WebResourcesPaths">
+    <contentEntries>
+      <entry url="file://$PROJECT_DIR$">
+        <entryData>
+          <resourceRoots>
+            <path value="file://$PROJECT_DIR$/ui" />
+          </resourceRoots>
+        </entryData>
+      </entry>
+    </contentEntries>
+  </component>
+</project>

.idea/yamllint.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="YamllintSettings">
+    <enabled>true</enabled>
+    <binPath>/usr/bin/yamllint</binPath>
+  </component>
+</project>

LICENSE.txt

@@ -2,14 +2,10 @@ Copyright (c) 2023, derfenix <derfenix@gmail.com> All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
-1) Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 
-2) Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 
-3) All advertising materials mentioning features or use of this software must display the following acknowledgement:
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 
-"This product includes software developed by the University of California, Berkeley and its contributors."
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-4) Neither the name of the <ORGANIZATION> nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -24,13 +24,17 @@ variables:
   * **LOGGING_DEBUG** — enable debug logs (default `false`)
 * **API**
   * **API_ADDRESS** — address the API server will listen (default `0.0.0.0:5001`)
+* **UI**
+  * **UI_ENABLED** — Enable builtin web UI (default `true`)
+  * **UI_PREFIX** — Prefix for the web UI (default `/`)
+  * **UI_THEME** — UI theme name (default `basic`). No other values available yet
 * **PDF**
   * **PDF_LANDSCAPE** — use landscape page orientation instead of portrait (default `false`)
   * **PDF_GRAYSCALE** — use grayscale filter for the output pdf (default `false`)
   * **PDF_MEDIA_PRINT** — use media type `print` for the request (default `true`)
   * **PDF_ZOOM** — zoom page (default `1.0` i.e. no actual zoom)
-  * **PDF_VIEWPORT** — use specified viewport value (default `1920x1080`)
+  * **PDF_VIEWPORT** — use specified viewport value (default `1280x720`)
-  * **PDF_DPI** — use specified DPI value for the output pdf (default `300`)
+  * **PDF_DPI** — use specified DPI value for the output pdf (default `150`)
   * **PDF_FILENAME** — use specified name for output pdf file (default `page.pdf`)
 
 
@@ -60,7 +64,7 @@ docker compose up -d webarchive
 ### 2. Add a page
 
 ```shell
-curl -X POST --location "http://localhost:5001/pages" \
+curl -X POST --location "http://localhost:5001/api/v1/pages" \
     -H "Content-Type: application/json" \
     -d "{
           \"url\": \"https://github.com/wkhtmltopdf/wkhtmltopdf/issues/1937\",
@@ -75,13 +79,13 @@ or
 
 ```shell
 curl -X POST --location \
-  "http://localhost:5001/pages?url=https%3A%2F%2Fgithub.com%2Fwkhtmltopdf%2Fwkhtmltopdf%2Fissues%2F1937&formats=pdf%2Cheaders&description=Foo+Bar"
+  "http://localhost:5001/api/v1/pages?url=https%3A%2F%2Fgithub.com%2Fwkhtmltopdf%2Fwkhtmltopdf%2Fissues%2F1937&formats=pdf%2Cheaders&description=Foo+Bar"
 ```
 
 ### 3. Get the page's info
 
 ```shell
-curl -X GET --location "http://localhost:5001/pages/$page_id" | jq .
+curl -X GET --location "http://localhost:5001/api/v1/pages/$page_id" | jq .
 ```
 where `$page_id` — value of the `id` field from previous command response.
 If `status` field in response is `success` (or `with_errors`) - the `results` field
@@ -90,7 +94,7 @@ will contain all processed formats with ids of the stored files.
 ### 4. Open file in browser
 
 ```shell
-xdg-open "http://localhost:5001/pages/$page_id/file/$file_id"
+xdg-open "http://localhost:5001/api/v1/pages/$page_id/file/$file_id"
 ```
 Where  `$page_id` — value of the `id` field from previous command response, and
 `$file_id` — the id of interesting file.
@@ -98,7 +102,7 @@ Where  `$page_id` — value of the `id` field from previous command response, an
 ### 5. List all stored pages
 
 ```shell
-curl -X GET --location "http://localhost:5001/pages" | jq .
+curl -X GET --location "http://localhost:5001/api/v1/pages" | jq .
 ```
 
 ## Roadmap
@@ -111,3 +115,5 @@ curl -X GET --location "http://localhost:5001/pages" | jq .
 - [ ] Optional authentication
 - [ ] Multi-user access
 - [ ] Support SQL database with or without separate files storage
+- [ ] Tags/Categories
+- [ ] Save page to markdown

adapters/processors/headers.go

@@ -17,7 +17,7 @@ type Headers struct {
 	client *http.Client
 }
 
-func (h *Headers) Process(ctx context.Context, url string) ([]entity.File, error) {
+func (h *Headers) Process(ctx context.Context, url string, _ *entity.Cache) ([]entity.File, error) {
 	var (
 		headersFile entity.File
 		err         error

adapters/processors/internal/mediainline.go

@@ -0,0 +1,255 @@
+package internal
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/disintegration/imaging"
+	"github.com/gabriel-vasile/mimetype"
+	"go.uber.org/zap"
+	"golang.org/x/net/html"
+)
+
+type MediaInline struct {
+	log    *zap.Logger
+	getter func(context.Context, string) (*http.Response, error)
+}
+
+func NewMediaInline(log *zap.Logger, getter func(context.Context, string) (*http.Response, error)) *MediaInline {
+	return &MediaInline{log: log, getter: getter}
+}
+
+func (m *MediaInline) Inline(ctx context.Context, reader io.Reader, pageURL string) (*html.Node, error) {
+	htmlNode, err := html.Parse(reader)
+	if err != nil {
+		return nil, fmt.Errorf("parse response body: %w", err)
+	}
+
+	baseURL, err := url.Parse(pageURL)
+	if err != nil {
+		return nil, fmt.Errorf("parse page url: %w", err)
+	}
+
+	m.visit(ctx, htmlNode, m.processorFunc, baseURL)
+
+	return htmlNode, nil
+}
+
+func (m *MediaInline) processorFunc(ctx context.Context, node *html.Node, baseURL *url.URL) error {
+	switch node.Data {
+	case "link":
+		if err := m.processHref(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process link %s: %w", node.Attr, err)
+		}
+
+	case "script", "img":
+		if err := m.processSrc(ctx, node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process script %s: %w", node.Attr, err)
+		}
+
+	case "a":
+		if err := m.processAHref(node.Attr, baseURL); err != nil {
+			return fmt.Errorf("process a href %s: %w", node.Attr, err)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processAHref(attrs []html.Attribute, baseURL *url.URL) error {
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "href":
+			attrs[idx].Val = normalizeURL(attr.Val, baseURL)
+		}
+	}
+
+	return nil
+}
+
+func (m *MediaInline) processHref(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "rel":
+			switch attr.Val {
+			case "stylesheet", "icon", "alternate icon", "shortcut icon", "manifest":
+				shouldProcess = true
+			}
+
+		case "href":
+			value = attr.Val
+			valueIdx = idx
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) processSrc(ctx context.Context, attrs []html.Attribute, baseURL *url.URL) error {
+	var shouldProcess bool
+	var value string
+	var valueIdx int
+
+	for idx, attr := range attrs {
+		switch attr.Key {
+		case "src":
+			value = attr.Val
+			valueIdx = idx
+			shouldProcess = true
+		case "data-src":
+			value = attr.Val
+		}
+	}
+
+	if !shouldProcess {
+		return nil
+	}
+
+	encodedValue, err := m.loadAndEncode(ctx, baseURL, value)
+	if err != nil {
+		return err
+	}
+
+	attrs[valueIdx].Val = encodedValue
+
+	return nil
+}
+
+func (m *MediaInline) loadAndEncode(ctx context.Context, baseURL *url.URL, value string) (string, error) {
+	mime := "text/plain"
+
+	if value == "" {
+		return "", nil
+	}
+
+	normalizedURL := normalizeURL(value, baseURL)
+	if normalizedURL == "" {
+		return value, nil
+	}
+
+	response, err := m.getter(ctx, normalizedURL)
+	if err != nil {
+		m.log.Sugar().With(zap.Error(err)).Errorf("load %s", normalizedURL)
+		return value, nil
+	}
+
+	defer func() {
+		_ = response.Body.Close()
+	}()
+
+	cleanMime := func(s string) string {
+		s, _, _ = strings.Cut(s, "+")
+		return s
+	}
+
+	if ct := response.Header.Get("Content-Type"); ct != "" {
+		mime = ct
+	}
+
+	encodedVal, err := m.encodeResource(response.Body, &mime)
+	if err != nil {
+		return value, fmt.Errorf("encode resource: %w", err)
+	}
+
+	return fmt.Sprintf("data:%s;base64, %s", cleanMime(mime), encodedVal), nil
+}
+
+func (m *MediaInline) visit(ctx context.Context, n *html.Node, proc func(context.Context, *html.Node, *url.URL) error, baseURL *url.URL) {
+	if err := proc(ctx, n, baseURL); err != nil {
+		m.log.Error("process error", zap.Error(err))
+	}
+
+	if n.FirstChild != nil {
+		m.visit(ctx, n.FirstChild, proc, baseURL)
+	}
+
+	if n.NextSibling != nil {
+		m.visit(ctx, n.NextSibling, proc, baseURL)
+	}
+}
+
+func normalizeURL(resourceURL string, base *url.URL) string {
+	if strings.HasPrefix(resourceURL, "//") {
+		return "https:" + resourceURL
+	}
+
+	if strings.HasPrefix(resourceURL, "about:") {
+		return ""
+	}
+
+	parsedResourceURL, err := url.Parse(resourceURL)
+	if err != nil {
+		return resourceURL
+	}
+
+	reference := base.ResolveReference(parsedResourceURL)
+
+	return reference.String()
+}
+
+func (m *MediaInline) encodeResource(r io.Reader, mime *string) (string, error) {
+	all, err := io.ReadAll(r)
+	if err != nil {
+		return "", fmt.Errorf("read data: %w", err)
+	}
+
+	all, err = m.preprocessResource(all, mime)
+	if err != nil {
+		return "", fmt.Errorf("preprocess resource: %w", err)
+	}
+
+	return base64.StdEncoding.EncodeToString(all), nil
+}
+
+func (m *MediaInline) preprocessResource(data []byte, mime *string) ([]byte, error) {
+	detectedMime := mimetype.Detect(data)
+
+	switch {
+	case strings.HasPrefix(detectedMime.String(), "image"):
+		decodedImage, err := imaging.Decode(bytes.NewBuffer(data))
+		if err != nil {
+			m.log.Error("failed to decode image", zap.Error(err))
+
+			return data, nil
+		}
+
+		if size := decodedImage.Bounds().Size(); size.X > 1024 || size.Y > 1024 {
+			thumbnail := imaging.Thumbnail(decodedImage, 1024, 1024, imaging.Lanczos)
+			buf := bytes.NewBuffer(nil)
+
+			if err := imaging.Encode(buf, thumbnail, imaging.JPEG, imaging.JPEGQuality(90)); err != nil {
+				m.log.Error("failed to create resized image", zap.Error(err))
+
+				return data, nil
+			}
+
+			*mime = "image/jpeg"
+			m.log.Info("Resized")
+
+			return buf.Bytes(), nil
+		}
+	}
+
+	return data, nil
+}

adapters/processors/pdf.go

@@ -19,7 +19,7 @@ type PDF struct {
 	cfg config.PDF
 }
 
-func (p *PDF) Process(_ context.Context, url string) ([]entity.File, error) {
+func (p *PDF) Process(_ context.Context, url string, cache *entity.Cache) ([]entity.File, error) {
 	gen, err := wkhtmltopdf.NewPDFGenerator()
 	if err != nil {
 		return nil, fmt.Errorf("new pdf generator: %w", err)
@@ -37,16 +37,29 @@ func (p *PDF) Process(_ context.Context, url string) ([]entity.File, error) {
 	gen.Grayscale.Set(p.cfg.Grayscale)
 	gen.Title.Set(url)
 
-	page := wkhtmltopdf.NewPage(url)
+	opts := wkhtmltopdf.NewPageOptions()
-	page.PrintMediaType.Set(p.cfg.MediaPrint)
+	opts.PrintMediaType.Set(p.cfg.MediaPrint)
-	page.JavascriptDelay.Set(200)
+	opts.JavascriptDelay.Set(200)
-	page.LoadMediaErrorHandling.Set("ignore")
+	opts.DisableJavascript.Set(false)
-	page.FooterRight.Set("[page]")
+	opts.LoadErrorHandling.Set("ignore")
-	page.HeaderLeft.Set(url)
+	opts.LoadMediaErrorHandling.Set("skip")
-	page.HeaderRight.Set(time.Now().Format(time.DateOnly))
+	opts.FooterRight.Set("[opts]")
-	page.FooterFontSize.Set(10)
+	opts.HeaderLeft.Set(url)
-	page.Zoom.Set(p.cfg.Zoom)
+	opts.HeaderRight.Set(time.Now().Format(time.DateOnly))
-	page.ViewportSize.Set(p.cfg.Viewport)
+	opts.FooterFontSize.Set(10)
+	opts.Zoom.Set(p.cfg.Zoom)
+	opts.ViewportSize.Set(p.cfg.Viewport)
+	opts.NoBackground.Set(true)
+	opts.DisableLocalFileAccess.Set(false)
+	opts.DisableExternalLinks.Set(false)
+	opts.DisableInternalLinks.Set(false)
+
+	var page wkhtmltopdf.PageProvider
+	if len(cache.Get()) > 0 {
+		page = &wkhtmltopdf.PageReader{Input: cache.Reader(), PageOptions: opts}
+	} else {
+		page = &wkhtmltopdf.Page{Input: url, PageOptions: opts}
+	}
 
 	gen.AddPage(page)
 

adapters/processors/processors.go

@@ -3,20 +3,27 @@ package processors
 import (
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"net/http/cookiejar"
+	"strings"
 	"time"
 
+	"go.uber.org/zap"
+	"golang.org/x/net/html"
+
 	"github.com/derfenix/webarchive/config"
 	"github.com/derfenix/webarchive/entity"
 )
 
+const defaultEncoding = "utf-8"
+
 type processor interface {
-	Process(ctx context.Context, url string) ([]entity.File, error)
+	Process(ctx context.Context, url string, cache *entity.Cache) ([]entity.File, error)
 }
 
-func NewProcessors(cfg config.Config) (*Processors, error) {
+func NewProcessors(cfg config.Config, log *zap.Logger) (*Processors, error) {
 	jar, err := cookiejar.New(&cookiejar.Options{
 		PublicSuffixList: nil,
 	})
@@ -52,10 +59,11 @@ func NewProcessors(cfg config.Config) (*Processors, error) {
 	}
 
 	procs := Processors{
+		client: httpClient,
 		processors: map[entity.Format]processor{
 			entity.FormatHeaders:    NewHeaders(httpClient),
 			entity.FormatPDF:        NewPDF(cfg.PDF),
-			entity.FormatSingleFile: NewSingleFile(httpClient),
+			entity.FormatSingleFile: NewSingleFile(httpClient, log),
 		},
 	}
 
@@ -64,9 +72,10 @@ func NewProcessors(cfg config.Config) (*Processors, error) {
 
 type Processors struct {
 	processors map[entity.Format]processor
+	client     *http.Client
 }
 
-func (p *Processors) Process(ctx context.Context, format entity.Format, url string) entity.Result {
+func (p *Processors) Process(ctx context.Context, format entity.Format, url string, cache *entity.Cache) entity.Result {
 	result := entity.Result{Format: format}
 
 	proc, ok := p.processors[format]
@@ -76,7 +85,7 @@ func (p *Processors) Process(ctx context.Context, format entity.Format, url stri
 		return result
 	}
 
-	files, err := proc.Process(ctx, url)
+	files, err := proc.Process(ctx, url, cache)
 	if err != nil {
 		result.Err = fmt.Errorf("process: %w", err)
 
@@ -93,3 +102,102 @@ func (p *Processors) OverrideProcessor(format entity.Format, proc processor) err
 
 	return nil
 }
+
+func (p *Processors) GetMeta(ctx context.Context, url string, cache *entity.Cache) (entity.Meta, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return entity.Meta{}, fmt.Errorf("new request: %w", err)
+	}
+
+	response, err := p.client.Do(req)
+	if err != nil {
+		return entity.Meta{}, fmt.Errorf("do request: %w", err)
+	}
+
+	if response.StatusCode != http.StatusOK {
+		return entity.Meta{}, fmt.Errorf("want status 200, got %d", response.StatusCode)
+	}
+
+	if response.Body == nil {
+		return entity.Meta{}, fmt.Errorf("empty response body")
+	}
+
+	defer func() {
+		_ = response.Body.Close()
+	}()
+
+	tee := io.TeeReader(response.Body, cache)
+
+	htmlNode, err := html.Parse(tee)
+	if err != nil {
+		return entity.Meta{}, fmt.Errorf("parse response body: %w", err)
+	}
+
+	var fc *html.Node
+	for fc = htmlNode.FirstChild; fc != nil && fc.Data != "html"; fc = fc.NextSibling {
+	}
+
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
+	fc = fc.NextSibling
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
+	for fc = fc.FirstChild; fc != nil && fc.Data != "head"; fc = fc.NextSibling {
+		fmt.Println(fc.Data)
+	}
+
+	if fc == nil {
+		return entity.Meta{}, fmt.Errorf("failed to find html tag")
+	}
+
+	meta := entity.Meta{}
+	getMetaData(fc, &meta)
+	meta.Encoding = encodingFromHeader(response.Header)
+
+	return meta, nil
+}
+
+func getMetaData(n *html.Node, meta *entity.Meta) {
+	if n == nil {
+		return
+	}
+
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		if c.Type == html.ElementNode && c.Data == "title" {
+			meta.Title = c.FirstChild.Data
+		}
+		if c.Type == html.ElementNode && c.Data == "meta" {
+			attrs := make(map[string]string)
+			for _, attr := range c.Attr {
+				attrs[attr.Key] = attr.Val
+			}
+
+			name, ok := attrs["name"]
+			if ok && name == "description" {
+				meta.Description = attrs["content"]
+			}
+		}
+
+		getMetaData(c, meta)
+	}
+}
+
+func encodingFromHeader(headers http.Header) string {
+	var foundEncoding bool
+	var encoding string
+
+	_, encoding, foundEncoding = strings.Cut(headers.Get("Content-Type"), "; ")
+	if foundEncoding {
+		_, encoding, foundEncoding = strings.Cut(encoding, "=")
+	}
+
+	if !foundEncoding {
+		encoding = defaultEncoding
+	}
+
+	return encoding
+}

adapters/processors/processors_test.go

@@ -0,0 +1,30 @@
+package processors
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+
+	"github.com/derfenix/webarchive/config"
+	"github.com/derfenix/webarchive/entity"
+)
+
+func TestProcessors_GetMeta(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	cfg, err := config.NewConfig(ctx)
+	require.NoError(t, err)
+
+	procs, err := NewProcessors(cfg, zaptest.NewLogger(t))
+	require.NoError(t, err)
+
+	cache := entity.NewCache()
+
+	meta, err := procs.GetMeta(ctx, "https://habr.com/ru/companies/wirenboard/articles/722718/", cache)
+	require.NoError(t, err)
+	assert.Equal(t, "Сколько стоит умный дом? Рассказываю, как строил свой и что получилось за 1000 руб./м² / Хабр", meta.Title)
+}

adapters/processors/singlefile.go

@@ -3,30 +3,57 @@ package processors
 import (
 	"bytes"
 	"context"
-	"encoding/base64"
 	"fmt"
-	"io"
 	"net/http"
-	"net/url"
-	"strings"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/html"
-	"golang.org/x/net/html/atom"
 
+	"github.com/derfenix/webarchive/adapters/processors/internal"
 	"github.com/derfenix/webarchive/entity"
 )
 
-const defaultEncoding = "utf-8"
+func NewSingleFile(client *http.Client, log *zap.Logger) *SingleFile {
-
+	return &SingleFile{client: client, log: log}
-func NewSingleFile(client *http.Client) *SingleFile {
-	return &SingleFile{client: client}
 }
 
 type SingleFile struct {
 	client *http.Client
+	log    *zap.Logger
 }
 
-func (s *SingleFile) Process(ctx context.Context, url string) ([]entity.File, error) {
+func (s *SingleFile) Process(ctx context.Context, pageURL string, cache *entity.Cache) ([]entity.File, error) {
+	reader := cache.Reader()
+
+	if reader == nil {
+		response, err := s.get(ctx, pageURL)
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			_ = response.Body.Close()
+		}()
+
+		reader = response.Body
+	}
+
+	inlinedHTML, err := internal.NewMediaInline(s.log, s.get).Inline(ctx, reader, pageURL)
+	if err != nil {
+		return nil, fmt.Errorf("inline media: %w", err)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	if err := html.Render(buf, inlinedHTML); err != nil {
+		return nil, fmt.Errorf("render result html: %w", err)
+	}
+
+	htmlFile := entity.NewFile("page.html", buf.Bytes())
+
+	return []entity.File{htmlFile}, nil
+}
+
+func (s *SingleFile) get(ctx context.Context, url string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("new request: %w", err)
@@ -45,180 +72,5 @@ func (s *SingleFile) Process(ctx context.Context, url string) ([]entity.File, er
 		return nil, fmt.Errorf("empty response body")
 	}
 
-	defer func() {
+	return response, nil
-		_ = response.Body.Close()
-	}()
-
-	htmlNode, err := html.Parse(response.Body)
-	if err != nil {
-		return nil, fmt.Errorf("parse response body: %w", err)
-	}
-
-	if err := s.crawl(ctx, htmlNode, baseURL(url), getEncoding(response)); err != nil {
-		return nil, fmt.Errorf("crawl: %w", err)
-	}
-
-	buf := bytes.NewBuffer(nil)
-	if err := html.Render(buf, htmlNode); err != nil {
-		return nil, fmt.Errorf("render result html: %w", err)
-	}
-
-	htmlFile := entity.NewFile("page.html", buf.Bytes())
-
-	return []entity.File{htmlFile}, nil
-}
-
-func (s *SingleFile) crawl(ctx context.Context, node *html.Node, baseURL string, encoding string) error {
-	if node.Data == "head" {
-		s.setCharset(node, encoding)
-	}
-
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		if child.Type == html.ElementNode {
-			if err := s.findAndReplaceResources(ctx, child, baseURL); err != nil {
-				return err
-			}
-		}
-
-		if err := s.crawl(ctx, child, baseURL, encoding); err != nil {
-			return fmt.Errorf("crawl child %s: %w", child.Data, err)
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) findAndReplaceResources(ctx context.Context, node *html.Node, baseURL string) error {
-	switch node.DataAtom {
-	case atom.Img, atom.Image, atom.Script, atom.Style:
-		err := s.replaceResource(ctx, node, baseURL)
-		if err != nil {
-			return err
-		}
-
-	case atom.Link:
-		for _, attribute := range node.Attr {
-			if attribute.Key == "rel" && (attribute.Val == "stylesheet") {
-				if err := s.replaceResource(ctx, node, baseURL); err != nil {
-					return err
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) replaceResource(ctx context.Context, node *html.Node, baseURL string) error {
-	for i, attribute := range node.Attr {
-		if attribute.Key == "src" || attribute.Key == "href" {
-			encoded, contentType, err := s.loadResource(ctx, attribute.Val, baseURL)
-			if err != nil {
-				return fmt.Errorf("load resource for %s: %w", node.Data, err)
-			}
-
-			if len(encoded) == 0 {
-				attribute.Val = ""
-
-			} else {
-				attribute.Val = fmt.Sprintf("data:%s;base64, %s", contentType, encoded)
-			}
-
-			node.Attr[i] = attribute
-		}
-	}
-
-	return nil
-}
-
-func (s *SingleFile) loadResource(ctx context.Context, val, baseURL string) ([]byte, string, error) {
-	if !strings.HasPrefix(val, "http://") && !strings.HasPrefix(val, "https://") {
-		var err error
-		val, err = url.JoinPath(baseURL, val)
-		if err != nil {
-			return nil, "", fmt.Errorf("join base path %s and url %s: %w", baseURL, val, err)
-		}
-		val, err = url.PathUnescape(val)
-		if err != nil {
-			return nil, "", fmt.Errorf("unescape path %s: %w", val, err)
-		}
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, val, nil)
-	if err != nil {
-		return nil, "", fmt.Errorf("new request: %w", err)
-	}
-
-	response, err := s.client.Do(req)
-	if err != nil {
-		return nil, "", fmt.Errorf("do request: %w", err)
-	}
-
-	defer func() {
-		if response.Body != nil {
-			_ = response.Body.Close()
-		}
-	}()
-
-	if response.StatusCode != http.StatusOK {
-		return []byte{}, "", nil
-	}
-
-	raw, err := io.ReadAll(response.Body)
-	if err != nil {
-		return nil, "", fmt.Errorf("read body: %w", err)
-	}
-
-	encoded := make([]byte, base64.StdEncoding.EncodedLen(len(raw)))
-	base64.StdEncoding.Encode(encoded, raw)
-
-	return encoded, response.Header.Get("Content-Type"), nil
-}
-
-func (s *SingleFile) setCharset(node *html.Node, encoding string) {
-	var charsetExists bool
-
-	for child := node.FirstChild; child != nil; child = child.NextSibling {
-		if child.Data == "meta" {
-			for _, attribute := range child.Attr {
-				if attribute.Key == "charset" {
-					charsetExists = true
-				}
-			}
-		}
-	}
-
-	if !charsetExists {
-		node.AppendChild(&html.Node{
-			Type:     html.ElementNode,
-			DataAtom: atom.Meta,
-			Data:     "meta",
-			Attr: []html.Attribute{
-				{
-					Key: "charset",
-					Val: encoding,
-				},
-			},
-		})
-	}
-}
-
-func baseURL(val string) string {
-	parsed, err := url.Parse(val)
-	if err != nil {
-		return val
-	}
-
-	return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host)
-}
-
-func getEncoding(response *http.Response) string {
-	_, encoding, found := strings.Cut(response.Header.Get("Content-Type"), "charset=")
-	if !found {
-		return defaultEncoding
-	}
-
-	encoding = strings.TrimSpace(encoding)
-
-	return encoding
 }

adapters/repository/badger.go

@@ -1,4 +1,4 @@
-package badger
+package repository
 
 import (
 	"errors"

adapters/repository/badger/page.go

@@ -8,6 +8,8 @@ import (
 	"github.com/dgraph-io/badger/v4"
 	"github.com/google/uuid"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/entity"
 )
 
@@ -24,7 +26,9 @@ type Page struct {
 }
 
 func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
-	page := entity.Page{ID: pageID}
+	page := entity.Page{}
+	page.ID = pageID
+
 	var file *entity.File
 
 	err := p.db.View(func(txn *badger.Txn) error {
@@ -44,9 +48,9 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil
 			return fmt.Errorf("get value: %w", err)
 		}
 
-		for i := range page.Results.Results() {
+		for i := range page.Results {
-			for j := range page.Results.Results()[i].Files {
+			for j := range page.Results[i].Files {
-				ff := &page.Results.Results()[i].Files[j]
+				ff := &page.Results[i].Files[j]
 
 				if ff.ID == fileID {
 					file = ff
@@ -64,18 +68,18 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil
 	return file, nil
 }
 
-func (p *Page) Save(_ context.Context, site *entity.Page) error {
+func (p *Page) Save(_ context.Context, page *entity.Page) error {
 	if p.db.IsClosed() {
-		return ErrDBClosed
+		return repository.ErrDBClosed
 	}
 
-	marshaled, err := marshal(site)
+	marshaled, err := marshal(page)
 	if err != nil {
 		return fmt.Errorf("marshal data: %w", err)
 	}
 
 	if err := p.db.Update(func(txn *badger.Txn) error {
-		if err := txn.Set(p.key(site), marshaled); err != nil {
+		if err := txn.Set(p.key(page), marshaled); err != nil {
 			return fmt.Errorf("put data: %w", err)
 		}
 
@@ -88,16 +92,17 @@ func (p *Page) Save(_ context.Context, site *entity.Page) error {
 }
 
 func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
-	site := entity.Page{ID: id}
+	page := entity.Page{}
+	page.ID = id
 
 	err := p.db.View(func(txn *badger.Txn) error {
-		data, err := txn.Get(p.key(&site))
+		data, err := txn.Get(p.key(&page))
 		if err != nil {
 			return fmt.Errorf("get data: %w", err)
 		}
 
 		err = data.Value(func(val []byte) error {
-			if err := unmarshal(val, &site); err != nil {
+			if err := unmarshal(val, &page); err != nil {
 				return fmt.Errorf("unmarshal data: %w", err)
 			}
 
@@ -113,7 +118,7 @@ func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
 		return nil, fmt.Errorf("view: %w", err)
 	}
 
-	return &site, nil
+	return &page, nil
 }
 
 func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
@@ -143,15 +148,55 @@ func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
 				return fmt.Errorf("get item: %w", err)
 			}
 
-			pages = append(pages, &entity.Page{
+			pages = append(pages, &page)
-				ID:          page.ID,
+		}
-				URL:         page.URL,
+
-				Description: page.Description,
+		return nil
-				Created:     page.Created,
+	})
-				Formats:     page.Formats,
+
-				Version:     page.Version,
+	if err != nil {
-				Status:      page.Status,
+		return nil, fmt.Errorf("view: %w", err)
+	}
+
+	sort.Slice(pages, func(i, j int) bool {
+		return pages[i].Created.After(pages[j].Created)
+	})
+
+	return pages, nil
+}
+
+func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
+	pages := make([]entity.Page, 0, 100)
+
+	err := p.db.View(func(txn *badger.Txn) error {
+		iterator := txn.NewIterator(badger.DefaultIteratorOptions)
+
+		defer iterator.Close()
+
+		for iterator.Seek(p.prefix); iterator.ValidForPrefix(p.prefix); iterator.Next() {
+			if err := ctx.Err(); err != nil {
+				return fmt.Errorf("context canceled: %w", err)
+			}
+
+			var page entity.Page
+
+			err := iterator.Item().Value(func(val []byte) error {
+				if err := unmarshal(val, &page); err != nil {
+					return fmt.Errorf("unmarshal: %w", err)
+				}
+
+				return nil
 			})
+
+			if err != nil {
+				return fmt.Errorf("get item: %w", err)
+			}
+
+			if page.Status == entity.StatusNew || page.Status == entity.StatusProcessing {
+				//goland:noinspection GoVetCopyLock
+				pages = append(pages, page) //nolint:govet // didn't touch the lock here
+			}
+
 		}
 
 		return nil

adapters/repository/badger/page_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/zap/zaptest"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/entity"
 )
 
@@ -31,7 +33,7 @@ func TestSite(t *testing.T) {
 
 	log := zaptest.NewLogger(t)
 
-	db, err := NewBadger(tempDir, log.Named("db"))
+	db, err := repository.NewBadger(tempDir, log.Named("db"))
 	require.NoError(t, err)
 
 	siteRepo, err := NewPage(db)
@@ -49,12 +51,16 @@ func TestSite(t *testing.T) {
 		storedSite, err := siteRepo.Get(ctx, site.ID)
 		require.NoError(t, err)
 
-		assert.Equal(t, site, storedSite)
+		assert.Equal(t, site.ID, storedSite.ID)
+		assert.Equal(t, site.URL, storedSite.URL)
+		assert.Equal(t, site.Status, storedSite.Status)
 
 		all, err := siteRepo.ListAll(ctx)
 		require.NoError(t, err)
 		require.Len(t, all, 1)
 
-		assert.Equal(t, site, all[0])
+		assert.Equal(t, site.ID, all[0].ID)
+		assert.Equal(t, site.URL, all[0].URL)
+		assert.Equal(t, site.Status, all[0].Status)
 	})
 }

adapters/repository/badgers3/marshal.go

@@ -0,0 +1,13 @@
+package badgers3
+
+import (
+	"github.com/vmihailenco/msgpack/v5"
+)
+
+func marshal(v interface{}) ([]byte, error) {
+	return msgpack.Marshal(v)
+}
+
+func unmarshal(b []byte, v interface{}) error { //nolint:unused // will use later
+	return msgpack.Unmarshal(b, v)
+}

adapters/repository/badgers3/page.go

@@ -0,0 +1,119 @@
+package badgers3
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/derfenix/webarchive/adapters/repository"
+	"github.com/derfenix/webarchive/entity"
+	"github.com/dgraph-io/badger/v4"
+	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
+)
+
+func NewPage(db *badger.DB, s3 *minio.Client, bucketName string) (*Page, error) {
+	return &Page{
+		db:         db,
+		s3:         s3,
+		prefix:     []byte("pages3:"),
+		bucketName: bucketName,
+	}, nil
+}
+
+type Page struct {
+	db         *badger.DB
+	s3         *minio.Client
+	prefix     []byte
+	bucketName string
+}
+
+func (p *Page) ListAll(ctx context.Context) ([]*entity.PageBase, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) Get(ctx context.Context, id uuid.UUID) (*entity.Page, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) GetFile(ctx context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) Save(ctx context.Context, page *entity.Page) error {
+	if p.db.IsClosed() {
+		return repository.ErrDBClosed
+	}
+
+	marshaled, err := marshal(page.PageBase)
+	if err != nil {
+		return fmt.Errorf("marshal data: %w", err)
+	}
+
+	if err := p.db.Update(func(txn *badger.Txn) error {
+		if err := txn.Set(p.key(page), marshaled); err != nil {
+			return fmt.Errorf("put data: %w", err)
+		}
+
+		return nil
+	}); err != nil {
+		return fmt.Errorf("update db: %w", err)
+	}
+
+	snowball := make(chan minio.SnowballObject, 1)
+
+	go func() {
+		defer close(snowball)
+
+		for _, result := range page.Results {
+			for _, file := range result.Files {
+				for {
+					if ctx.Err() != nil {
+						return
+					}
+
+					if len(snowball) < cap(snowball) {
+						break
+					}
+				}
+
+				snowball <- minio.SnowballObject{
+					Key:     file.ID.String(),
+					Size:    int64(len(file.Data)),
+					ModTime: time.Now(),
+					Content: bytes.NewReader(file.Data),
+				}
+			}
+		}
+	}()
+
+	if err = p.s3.PutObjectsSnowball(ctx, p.bucketName, minio.SnowballOptions{Compress: true}, snowball); err != nil {
+		if dErr := p.db.Update(func(txn *badger.Txn) error {
+			if err := txn.Delete(p.key(page)); err != nil {
+				return fmt.Errorf("put data: %w", err)
+			}
+
+			return nil
+		}); dErr != nil {
+			err = errors.Join(err, dErr)
+		}
+
+		return fmt.Errorf("store files to s3: %w", err)
+	}
+
+	return nil
+}
+
+func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (p *Page) key(site *entity.Page) []byte {
+	return append(p.prefix, []byte(site.ID.String())...)
+}

api/gen.go

@@ -1,3 +1,3 @@
 package api
 
-//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.60.1 --target ./openapi -package openapi --clean openapi.yaml
+//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.77.0 --target ./openapi -package openapi --clean openapi.yaml

api/openapi.yaml

@@ -1,10 +1,11 @@
-openapi: 3.1.0
+---
+openapi: 3.0.3
 info:
   title: Sample API
   description: API description in Markdown.
   version: 1.0.0
 servers:
-  - url: 'https://api.example.com'
+  - url: 'https://api.example.com/api/v1'
 paths:
   /pages:
     get:
@@ -125,7 +126,7 @@ paths:
         200:
           description: File content
           content:
-            application/pdf: { }
+            application/pdf: {}
             text/plain:
               schema:
                 type: string
@@ -183,12 +184,25 @@ components:
             $ref: '#/components/schemas/format'
         status:
           $ref: '#/components/schemas/status'
+        meta:
+          type: object
+          properties:
+            title:
+              type: string
+            description:
+              type: string
+            error:
+              type: string
+          required:
+            - title
+            - description
       required:
         - id
         - url
         - formats
         - status
         - created
+        - meta
     result:
       type: object
       properties:

api/openapi/oas_cfg_gen.go

@@ -7,7 +7,6 @@ import (
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/metric/instrument"
 	"go.opentelemetry.io/otel/trace"
 
 	ht "github.com/ogen-go/ogen/http"
@@ -40,7 +39,7 @@ func (cfg *otelConfig) initOTEL() {
 		cfg.TracerProvider = otel.GetTracerProvider()
 	}
 	if cfg.MeterProvider == nil {
-		cfg.MeterProvider = metric.NewNoopMeterProvider()
+		cfg.MeterProvider = otel.GetMeterProvider()
 	}
 	cfg.Tracer = cfg.TracerProvider.Tracer(otelogen.Name,
 		trace.WithInstrumentationVersion(otelogen.SemVersion()),
@@ -99,9 +98,9 @@ func newServerConfig(opts ...ServerOption) serverConfig {
 
 type baseServer struct {
 	cfg      serverConfig
-	requests instrument.Int64Counter
+	requests metric.Int64Counter
-	errors   instrument.Int64Counter
+	errors   metric.Int64Counter
-	duration instrument.Int64Histogram
+	duration metric.Float64Histogram
 }
 
 func (s baseServer) notFound(w http.ResponseWriter, r *http.Request) {
@@ -120,7 +119,7 @@ func (cfg serverConfig) baseServer() (s baseServer, err error) {
 	if s.errors, err = s.cfg.Meter.Int64Counter(otelogen.ServerErrorsCount); err != nil {
 		return s, err
 	}
-	if s.duration, err = s.cfg.Meter.Int64Histogram(otelogen.ServerDuration); err != nil {
+	if s.duration, err = s.cfg.Meter.Float64Histogram(otelogen.ServerDuration); err != nil {
 		return s, err
 	}
 	return s, nil
@@ -162,9 +161,9 @@ func newClientConfig(opts ...ClientOption) clientConfig {
 
 type baseClient struct {
 	cfg      clientConfig
-	requests instrument.Int64Counter
+	requests metric.Int64Counter
-	errors   instrument.Int64Counter
+	errors   metric.Int64Counter
-	duration instrument.Int64Histogram
+	duration metric.Float64Histogram
 }
 
 func (cfg clientConfig) baseClient() (c baseClient, err error) {
@@ -175,7 +174,7 @@ func (cfg clientConfig) baseClient() (c baseClient, err error) {
 	if c.errors, err = c.cfg.Meter.Int64Counter(otelogen.ClientErrorsCount); err != nil {
 		return c, err
 	}
-	if c.duration, err = c.cfg.Meter.Int64Histogram(otelogen.ClientDuration); err != nil {
+	if c.duration, err = c.cfg.Meter.Float64Histogram(otelogen.ClientDuration); err != nil {
 		return c, err
 	}
 	return c, nil
@@ -200,7 +199,7 @@ func WithTracerProvider(provider trace.TracerProvider) Option {
 
 // WithMeterProvider specifies a meter provider to use for creating a meter.
 //
-// If none is specified, the metric.NewNoopMeterProvider is used.
+// If none is specified, the otel.GetMeterProvider() is used.
 func WithMeterProvider(provider metric.MeterProvider) Option {
 	return otelOptionFunc(func(cfg *otelConfig) {
 		if provider != nil {

api/openapi/oas_client_gen.go

@@ -11,6 +11,8 @@ import (
 	"github.com/go-faster/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/ogen-go/ogen/conv"
@@ -19,6 +21,34 @@ import (
 	"github.com/ogen-go/ogen/uri"
 )
 
+// Invoker invokes operations described by OpenAPI v3 specification.
+type Invoker interface {
+	// AddPage invokes addPage operation.
+	//
+	// Add new page.
+	//
+	// POST /pages
+	AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error)
+	// GetFile invokes getFile operation.
+	//
+	// Get file content.
+	//
+	// GET /pages/{id}/file/{file_id}
+	GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error)
+	// GetPage invokes getPage operation.
+	//
+	// Get page details.
+	//
+	// GET /pages/{id}
+	GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error)
+	// GetPages invokes getPages operation.
+	//
+	// Get all pages.
+	//
+	// GET /pages
+	GetPages(ctx context.Context) (Pages, error)
+}
+
 // Client implements OAS client.
 type Client struct {
 	serverURL *url.URL
@@ -78,19 +108,20 @@ func (c *Client) requestURL(ctx context.Context) *url.URL {
 // POST /pages
 func (c *Client) AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error) {
 	res, err := c.sendAddPage(ctx, request, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (res AddPageRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("addPage"),
+		semconv.HTTPMethodKey.String("POST"),
+		semconv.HTTPRouteKey.String("/pages"),
 	}
 	// Validate request before sending.
 	if err := func() error {
-		if request.Set {
+		if value, ok := request.Get(); ok {
 			if err := func() error {
-				if err := request.Value.Validate(); err != nil {
+				if err := value.Validate(); err != nil {
 					return err
 				}
 				return nil
@@ -106,12 +137,13 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "AddPage",
@@ -124,7 +156,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -197,7 +229,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 	u.RawQuery = q.Values().Encode()
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "POST", u, nil)
+	r, err := ht.NewRequest(ctx, "POST", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -228,24 +260,26 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
 // GET /pages/{id}/file/{file_id}
 func (c *Client) GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error) {
 	res, err := c.sendGetFile(ctx, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res GetFileRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getFile"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages/{id}/file/{file_id}"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetFile",
@@ -258,7 +292,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -307,7 +341,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -335,24 +369,26 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
 // GET /pages/{id}
 func (c *Client) GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error) {
 	res, err := c.sendGetPage(ctx, params)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res GetPageRes, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getPage"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages/{id}"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetPage",
@@ -365,7 +401,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -395,7 +431,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}
@@ -423,24 +459,26 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
 // GET /pages
 func (c *Client) GetPages(ctx context.Context) (Pages, error) {
 	res, err := c.sendGetPages(ctx)
-	_ = res
 	return res, err
 }
 
 func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 	otelAttrs := []attribute.KeyValue{
 		otelogen.OperationID("getPages"),
+		semconv.HTTPMethodKey.String("GET"),
+		semconv.HTTPRouteKey.String("/pages"),
 	}
 
 	// Run stopwatch.
 	startTime := time.Now()
 	defer func() {
+		// Use floating point division here for higher precision (instead of Millisecond method).
 		elapsedDuration := time.Since(startTime)
-		c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	c.requests.Add(ctx, 1, otelAttrs...)
+	c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	// Start a span for this request.
 	ctx, span := c.cfg.Tracer.Start(ctx, "GetPages",
@@ -453,7 +491,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 		if err != nil {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			c.errors.Add(ctx, 1, otelAttrs...)
+			c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		span.End()
 	}()
@@ -465,7 +503,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
 	uri.AddPathParts(u, pathParts[:]...)
 
 	stage = "EncodeRequest"
-	r, err := ht.NewRequest(ctx, "GET", u, nil)
+	r, err := ht.NewRequest(ctx, "GET", u)
 	if err != nil {
 		return res, errors.Wrap(err, "create request")
 	}

api/openapi/oas_handlers_gen.go

@@ -10,7 +10,8 @@ import (
 	"github.com/go-faster/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
-	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
+	"go.opentelemetry.io/otel/metric"
+	semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
 	"go.opentelemetry.io/otel/trace"
 
 	ht "github.com/ogen-go/ogen/http"
@@ -42,17 +43,18 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -89,10 +91,11 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 	var response AddPageRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
+			Context:          ctx,
-			OperationName: "AddPage",
+			OperationName:    "AddPage",
-			OperationID:   "addPage",
+			OperationSummary: "Add new page",
-			Body:          request,
+			OperationID:      "addPage",
+			Body:             request,
 			Params: middleware.Parameters{
 				{
 					Name: "url",
@@ -132,22 +135,27 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
 		response, err = s.h.AddPage(ctx, request, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeAddPageResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -175,17 +183,18 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -207,10 +216,11 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 	var response GetFileRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
+			Context:          ctx,
-			OperationName: "GetFile",
+			OperationName:    "GetFile",
-			OperationID:   "getFile",
+			OperationSummary: "",
-			Body:          nil,
+			OperationID:      "getFile",
+			Body:             nil,
 			Params: middleware.Parameters{
 				{
 					Name: "id",
@@ -246,22 +256,27 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
 		response, err = s.h.GetFile(ctx, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetFileResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -289,17 +304,18 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err          error
 		opErrContext = ogenerrors.OperationContext{
@@ -321,10 +337,11 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 	var response GetPageRes
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
+			Context:          ctx,
-			OperationName: "GetPage",
+			OperationName:    "GetPage",
-			OperationID:   "getPage",
+			OperationSummary: "",
-			Body:          nil,
+			OperationID:      "getPage",
+			Body:             nil,
 			Params: middleware.Parameters{
 				{
 					Name: "id",
@@ -356,22 +373,27 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
 		response, err = s.h.GetPage(ctx, params)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetPageResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }
@@ -399,17 +421,18 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 	startTime := time.Now()
 	defer func() {
 		elapsedDuration := time.Since(startTime)
-		s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
+		// Use floating point division here for higher precision (instead of Millisecond method).
+		s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
 	}()
 
 	// Increment request counter.
-	s.requests.Add(ctx, 1, otelAttrs...)
+	s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 
 	var (
 		recordError = func(stage string, err error) {
 			span.RecordError(err)
 			span.SetStatus(codes.Error, stage)
-			s.errors.Add(ctx, 1, otelAttrs...)
+			s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
 		}
 		err error
 	)
@@ -417,12 +440,13 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 	var response Pages
 	if m := s.cfg.Middleware; m != nil {
 		mreq := middleware.Request{
-			Context:       ctx,
+			Context:          ctx,
-			OperationName: "GetPages",
+			OperationName:    "GetPages",
-			OperationID:   "getPages",
+			OperationSummary: "Get all pages",
-			Body:          nil,
+			OperationID:      "getPages",
-			Params:        middleware.Parameters{},
+			Body:             nil,
-			Raw:           r,
+			Params:           middleware.Parameters{},
+			Raw:              r,
 		}
 
 		type (
@@ -447,22 +471,27 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
 		response, err = s.h.GetPages(ctx)
 	}
 	if err != nil {
-		recordError("Internal", err)
 		if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
-			encodeErrorResponse(errRes, w, span)
+			if err := encodeErrorResponse(errRes, w, span); err != nil {
+				recordError("Internal", err)
+			}
 			return
 		}
 		if errors.Is(err, ht.ErrNotImplemented) {
 			s.cfg.ErrorHandler(ctx, w, r, err)
 			return
 		}
-		encodeErrorResponse(s.h.NewError(ctx, err), w, span)
+		if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
+			recordError("Internal", err)
+		}
 		return
 	}
 
 	if err := encodeGetPagesResponse(response, w, span); err != nil {
 		recordError("EncodeResponse", err)
-		s.cfg.ErrorHandler(ctx, w, r, err)
+		if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
+			s.cfg.ErrorHandler(ctx, w, r, err)
+		}
 		return
 	}
 }

api/openapi/oas_json_gen.go

@@ -23,12 +23,10 @@ func (s *AddPageBadRequest) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *AddPageBadRequest) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("field")
 		e.Str(s.Field)
 	}
 	{
-
 		e.FieldStart("error")
 		e.Str(s.Error)
 	}
@@ -138,7 +136,6 @@ func (s *AddPageReq) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *AddPageReq) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
@@ -280,7 +277,6 @@ func (s *Error) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Error) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("message")
 		e.Str(s.Message)
 	}
@@ -506,22 +502,18 @@ func (s *Page) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Page) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
 	{
-
 		e.FieldStart("created")
 		json.EncodeDateTime(e, s.Created)
 	}
 	{
-
 		e.FieldStart("formats")
 		e.ArrStart()
 		for _, elem := range s.Formats {
@@ -530,18 +522,22 @@ func (s *Page) encodeFields(e *jx.Encoder) {
 		e.ArrEnd()
 	}
 	{
-
 		e.FieldStart("status")
 		s.Status.Encode(e)
 	}
+	{
+		e.FieldStart("meta")
+		s.Meta.Encode(e)
+	}
 }
 
-var jsonFieldsNameOfPage = [5]string{
+var jsonFieldsNameOfPage = [6]string{
 	0: "id",
 	1: "url",
 	2: "created",
 	3: "formats",
 	4: "status",
+	5: "meta",
 }
 
 // Decode decodes Page from json.
@@ -617,6 +613,16 @@ func (s *Page) Decode(d *jx.Decoder) error {
 			}(); err != nil {
 				return errors.Wrap(err, "decode field \"status\"")
 			}
+		case "meta":
+			requiredBitSet[0] |= 1 << 5
+			if err := func() error {
+				if err := s.Meta.Decode(d); err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"meta\"")
+			}
 		default:
 			return d.Skip()
 		}
@@ -627,7 +633,7 @@ func (s *Page) Decode(d *jx.Decoder) error {
 	// Validate required fields.
 	var failures []validate.FieldError
 	for i, mask := range [1]uint8{
-		0b00011111,
+		0b00111111,
 	} {
 		if result := (requiredBitSet[i] & mask) ^ mask; result != 0 {
 			// Mask only required fields and check equality to mask using XOR.
@@ -673,6 +679,136 @@ func (s *Page) UnmarshalJSON(data []byte) error {
 	return s.Decode(d)
 }
 
+// Encode implements json.Marshaler.
+func (s *PageMeta) Encode(e *jx.Encoder) {
+	e.ObjStart()
+	s.encodeFields(e)
+	e.ObjEnd()
+}
+
+// encodeFields encodes fields.
+func (s *PageMeta) encodeFields(e *jx.Encoder) {
+	{
+		e.FieldStart("title")
+		e.Str(s.Title)
+	}
+	{
+		e.FieldStart("description")
+		e.Str(s.Description)
+	}
+	{
+		if s.Error.Set {
+			e.FieldStart("error")
+			s.Error.Encode(e)
+		}
+	}
+}
+
+var jsonFieldsNameOfPageMeta = [3]string{
+	0: "title",
+	1: "description",
+	2: "error",
+}
+
+// Decode decodes PageMeta from json.
+func (s *PageMeta) Decode(d *jx.Decoder) error {
+	if s == nil {
+		return errors.New("invalid: unable to decode PageMeta to nil")
+	}
+	var requiredBitSet [1]uint8
+
+	if err := d.ObjBytes(func(d *jx.Decoder, k []byte) error {
+		switch string(k) {
+		case "title":
+			requiredBitSet[0] |= 1 << 0
+			if err := func() error {
+				v, err := d.Str()
+				s.Title = string(v)
+				if err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"title\"")
+			}
+		case "description":
+			requiredBitSet[0] |= 1 << 1
+			if err := func() error {
+				v, err := d.Str()
+				s.Description = string(v)
+				if err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"description\"")
+			}
+		case "error":
+			if err := func() error {
+				s.Error.Reset()
+				if err := s.Error.Decode(d); err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"error\"")
+			}
+		default:
+			return d.Skip()
+		}
+		return nil
+	}); err != nil {
+		return errors.Wrap(err, "decode PageMeta")
+	}
+	// Validate required fields.
+	var failures []validate.FieldError
+	for i, mask := range [1]uint8{
+		0b00000011,
+	} {
+		if result := (requiredBitSet[i] & mask) ^ mask; result != 0 {
+			// Mask only required fields and check equality to mask using XOR.
+			//
+			// If XOR result is not zero, result is not equal to expected, so some fields are missed.
+			// Bits of fields which would be set are actually bits of missed fields.
+			missed := bits.OnesCount8(result)
+			for bitN := 0; bitN < missed; bitN++ {
+				bitIdx := bits.TrailingZeros8(result)
+				fieldIdx := i*8 + bitIdx
+				var name string
+				if fieldIdx < len(jsonFieldsNameOfPageMeta) {
+					name = jsonFieldsNameOfPageMeta[fieldIdx]
+				} else {
+					name = strconv.Itoa(fieldIdx)
+				}
+				failures = append(failures, validate.FieldError{
+					Name:  name,
+					Error: validate.ErrFieldRequired,
+				})
+				// Reset bit.
+				result &^= 1 << bitIdx
+			}
+		}
+	}
+	if len(failures) > 0 {
+		return &validate.Error{Fields: failures}
+	}
+
+	return nil
+}
+
+// MarshalJSON implements stdjson.Marshaler.
+func (s *PageMeta) MarshalJSON() ([]byte, error) {
+	e := jx.Encoder{}
+	s.Encode(&e)
+	return e.Bytes(), nil
+}
+
+// UnmarshalJSON implements stdjson.Unmarshaler.
+func (s *PageMeta) UnmarshalJSON(data []byte) error {
+	d := jx.DecodeBytes(data)
+	return s.Decode(d)
+}
+
 // Encode implements json.Marshaler.
 func (s *PageWithResults) Encode(e *jx.Encoder) {
 	e.ObjStart()
@@ -683,22 +819,18 @@ func (s *PageWithResults) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *PageWithResults) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("url")
 		e.Str(s.URL)
 	}
 	{
-
 		e.FieldStart("created")
 		json.EncodeDateTime(e, s.Created)
 	}
 	{
-
 		e.FieldStart("formats")
 		e.ArrStart()
 		for _, elem := range s.Formats {
@@ -707,12 +839,14 @@ func (s *PageWithResults) encodeFields(e *jx.Encoder) {
 		e.ArrEnd()
 	}
 	{
-
 		e.FieldStart("status")
 		s.Status.Encode(e)
 	}
 	{
-
+		e.FieldStart("meta")
+		s.Meta.Encode(e)
+	}
+	{
 		e.FieldStart("results")
 		e.ArrStart()
 		for _, elem := range s.Results {
@@ -722,13 +856,14 @@ func (s *PageWithResults) encodeFields(e *jx.Encoder) {
 	}
 }
 
-var jsonFieldsNameOfPageWithResults = [6]string{
+var jsonFieldsNameOfPageWithResults = [7]string{
 	0: "id",
 	1: "url",
 	2: "created",
 	3: "formats",
 	4: "status",
-	5: "results",
+	5: "meta",
+	6: "results",
 }
 
 // Decode decodes PageWithResults from json.
@@ -804,8 +939,18 @@ func (s *PageWithResults) Decode(d *jx.Decoder) error {
 			}(); err != nil {
 				return errors.Wrap(err, "decode field \"status\"")
 			}
-		case "results":
+		case "meta":
 			requiredBitSet[0] |= 1 << 5
+			if err := func() error {
+				if err := s.Meta.Decode(d); err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"meta\"")
+			}
+		case "results":
+			requiredBitSet[0] |= 1 << 6
 			if err := func() error {
 				s.Results = make([]Result, 0)
 				if err := d.Arr(func(d *jx.Decoder) error {
@@ -832,7 +977,7 @@ func (s *PageWithResults) Decode(d *jx.Decoder) error {
 	// Validate required fields.
 	var failures []validate.FieldError
 	for i, mask := range [1]uint8{
-		0b00111111,
+		0b01111111,
 	} {
 		if result := (requiredBitSet[i] & mask) ^ mask; result != 0 {
 			// Mask only required fields and check equality to mask using XOR.
@@ -878,6 +1023,136 @@ func (s *PageWithResults) UnmarshalJSON(data []byte) error {
 	return s.Decode(d)
 }
 
+// Encode implements json.Marshaler.
+func (s *PageWithResultsMeta) Encode(e *jx.Encoder) {
+	e.ObjStart()
+	s.encodeFields(e)
+	e.ObjEnd()
+}
+
+// encodeFields encodes fields.
+func (s *PageWithResultsMeta) encodeFields(e *jx.Encoder) {
+	{
+		e.FieldStart("title")
+		e.Str(s.Title)
+	}
+	{
+		e.FieldStart("description")
+		e.Str(s.Description)
+	}
+	{
+		if s.Error.Set {
+			e.FieldStart("error")
+			s.Error.Encode(e)
+		}
+	}
+}
+
+var jsonFieldsNameOfPageWithResultsMeta = [3]string{
+	0: "title",
+	1: "description",
+	2: "error",
+}
+
+// Decode decodes PageWithResultsMeta from json.
+func (s *PageWithResultsMeta) Decode(d *jx.Decoder) error {
+	if s == nil {
+		return errors.New("invalid: unable to decode PageWithResultsMeta to nil")
+	}
+	var requiredBitSet [1]uint8
+
+	if err := d.ObjBytes(func(d *jx.Decoder, k []byte) error {
+		switch string(k) {
+		case "title":
+			requiredBitSet[0] |= 1 << 0
+			if err := func() error {
+				v, err := d.Str()
+				s.Title = string(v)
+				if err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"title\"")
+			}
+		case "description":
+			requiredBitSet[0] |= 1 << 1
+			if err := func() error {
+				v, err := d.Str()
+				s.Description = string(v)
+				if err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"description\"")
+			}
+		case "error":
+			if err := func() error {
+				s.Error.Reset()
+				if err := s.Error.Decode(d); err != nil {
+					return err
+				}
+				return nil
+			}(); err != nil {
+				return errors.Wrap(err, "decode field \"error\"")
+			}
+		default:
+			return d.Skip()
+		}
+		return nil
+	}); err != nil {
+		return errors.Wrap(err, "decode PageWithResultsMeta")
+	}
+	// Validate required fields.
+	var failures []validate.FieldError
+	for i, mask := range [1]uint8{
+		0b00000011,
+	} {
+		if result := (requiredBitSet[i] & mask) ^ mask; result != 0 {
+			// Mask only required fields and check equality to mask using XOR.
+			//
+			// If XOR result is not zero, result is not equal to expected, so some fields are missed.
+			// Bits of fields which would be set are actually bits of missed fields.
+			missed := bits.OnesCount8(result)
+			for bitN := 0; bitN < missed; bitN++ {
+				bitIdx := bits.TrailingZeros8(result)
+				fieldIdx := i*8 + bitIdx
+				var name string
+				if fieldIdx < len(jsonFieldsNameOfPageWithResultsMeta) {
+					name = jsonFieldsNameOfPageWithResultsMeta[fieldIdx]
+				} else {
+					name = strconv.Itoa(fieldIdx)
+				}
+				failures = append(failures, validate.FieldError{
+					Name:  name,
+					Error: validate.ErrFieldRequired,
+				})
+				// Reset bit.
+				result &^= 1 << bitIdx
+			}
+		}
+	}
+	if len(failures) > 0 {
+		return &validate.Error{Fields: failures}
+	}
+
+	return nil
+}
+
+// MarshalJSON implements stdjson.Marshaler.
+func (s *PageWithResultsMeta) MarshalJSON() ([]byte, error) {
+	e := jx.Encoder{}
+	s.Encode(&e)
+	return e.Bytes(), nil
+}
+
+// UnmarshalJSON implements stdjson.Unmarshaler.
+func (s *PageWithResultsMeta) UnmarshalJSON(data []byte) error {
+	d := jx.DecodeBytes(data)
+	return s.Decode(d)
+}
+
 // Encode encodes Pages as json.
 func (s Pages) Encode(e *jx.Encoder) {
 	unwrapped := []Page(s)
@@ -938,7 +1213,6 @@ func (s *Result) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *Result) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("format")
 		s.Format.Encode(e)
 	}
@@ -949,7 +1223,6 @@ func (s *Result) encodeFields(e *jx.Encoder) {
 		}
 	}
 	{
-
 		e.FieldStart("files")
 		e.ArrStart()
 		for _, elem := range s.Files {
@@ -1078,22 +1351,18 @@ func (s *ResultFilesItem) Encode(e *jx.Encoder) {
 // encodeFields encodes fields.
 func (s *ResultFilesItem) encodeFields(e *jx.Encoder) {
 	{
-
 		e.FieldStart("id")
 		json.EncodeUUID(e, s.ID)
 	}
 	{
-
 		e.FieldStart("name")
 		e.Str(s.Name)
 	}
 	{
-
 		e.FieldStart("mimetype")
 		e.Str(s.Mimetype)
 	}
 	{
-
 		e.FieldStart("size")
 		e.Int64(s.Size)
 	}

api/openapi/oas_request_decoders_gen.go

@@ -77,9 +77,9 @@ func (s *Server) decodeAddPageRequest(r *http.Request) (
 			return req, close, err
 		}
 		if err := func() error {
-			if request.Set {
+			if value, ok := request.Get(); ok {
 				if err := func() error {
-					if err := request.Value.Validate(); err != nil {
+					if err := value.Validate(); err != nil {
 						return err
 					}
 					return nil

api/openapi/oas_request_encoders_gen.go

@@ -20,7 +20,7 @@ func encodeAddPageRequest(
 		// Keep request with empty body if value is not set.
 		return nil
 	}
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	{
 		if req.Set {
 			req.Encode(e)

api/openapi/oas_response_decoders_gen.go

@@ -15,7 +15,7 @@ import (
 	"github.com/ogen-go/ogen/validate"
 )
 
-func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
+func decodeAddPageResponse(resp *http.Response) (res AddPageRes, _ error) {
 	switch resp.StatusCode {
 	case 201:
 		// Code 201.
@@ -128,12 +128,12 @@ func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
+func decodeGetFileResponse(resp *http.Response) (res GetFileRes, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -216,12 +216,12 @@ func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
+func decodeGetPageResponse(resp *http.Response) (res GetPageRes, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -302,12 +302,12 @@ func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }
 
-func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
+func decodeGetPagesResponse(resp *http.Response) (res Pages, _ error) {
 	switch resp.StatusCode {
 	case 200:
 		// Code 200.
@@ -385,7 +385,7 @@ func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
 		}
 	}()
 	if err != nil {
-		return res, errors.Wrap(err, "default")
+		return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
 	}
 	return res, errors.Wrap(defRes, "error")
 }

api/openapi/oas_response_encoders_gen.go

@@ -10,6 +10,8 @@ import (
 	"github.com/go-faster/jx"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
+
+	ht "github.com/ogen-go/ogen/http"
 )
 
 func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trace.Span) error {
@@ -19,11 +21,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(201)
 		span.SetStatus(codes.Ok, http.StatusText(201))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *AddPageBadRequest:
@@ -31,11 +34,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(400)
 		span.SetStatus(codes.Error, http.StatusText(400))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	default:
@@ -54,6 +58,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileOKTextHTML:
@@ -65,6 +70,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileOKTextPlain:
@@ -76,6 +82,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
 		if _, err := io.Copy(writer, response); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetFileNotFound:
@@ -96,11 +103,12 @@ func encodeGetPageResponse(response GetPageRes, w http.ResponseWriter, span trac
 		w.WriteHeader(200)
 		span.SetStatus(codes.Ok, http.StatusText(200))
 
-		e := jx.GetEncoder()
+		e := new(jx.Encoder)
 		response.Encode(e)
 		if _, err := e.WriteTo(w); err != nil {
 			return errors.Wrap(err, "write")
 		}
+
 		return nil
 
 	case *GetPageNotFound:
@@ -119,11 +127,12 @@ func encodeGetPagesResponse(response Pages, w http.ResponseWriter, span trace.Sp
 	w.WriteHeader(200)
 	span.SetStatus(codes.Ok, http.StatusText(200))
 
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	response.Encode(e)
 	if _, err := e.WriteTo(w); err != nil {
 		return errors.Wrap(err, "write")
 	}
+
 	return nil
 }
 
@@ -142,11 +151,15 @@ func encodeErrorResponse(response *ErrorStatusCode, w http.ResponseWriter, span
 		span.SetStatus(codes.Ok, st)
 	}
 
-	e := jx.GetEncoder()
+	e := new(jx.Encoder)
 	response.Response.Encode(e)
 	if _, err := e.WriteTo(w); err != nil {
 		return errors.Wrap(err, "write")
 	}
+
+	if code >= http.StatusInternalServerError {
+		return errors.Wrapf(ht.ErrInternalServerErrorResponse, "code: %d, message: %s", code, http.StatusText(code))
+	}
 	return nil
 
 }

api/openapi/oas_router_gen.go

@@ -10,6 +10,19 @@ import (
 	"github.com/ogen-go/ogen/uri"
 )
 
+func (s *Server) cutPrefix(path string) (string, bool) {
+	prefix := s.cfg.Prefix
+	if prefix == "" {
+		return path, true
+	}
+	if !strings.HasPrefix(path, prefix) {
+		// Prefix doesn't match.
+		return "", false
+	}
+	// Cut prefix from the path.
+	return strings.TrimPrefix(path, prefix), true
+}
+
 // ServeHTTP serves http request as defined by OpenAPI v3 specification,
 // calling handler that matches the path or returning not found error.
 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -21,17 +34,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			elemIsEscaped = strings.ContainsRune(elem, '%')
 		}
 	}
-	if prefix := s.cfg.Prefix; len(prefix) > 0 {
+
-		if strings.HasPrefix(elem, prefix) {
+	elem, ok := s.cutPrefix(elem)
-			// Cut prefix from the path.
+	if !ok || len(elem) == 0 {
-			elem = strings.TrimPrefix(elem, prefix)
-		} else {
-			// Prefix doesn't match.
-			s.notFound(w, r)
-			return
-		}
-	}
-	if len(elem) == 0 {
 		s.notFound(w, r)
 		return
 	}
@@ -129,6 +134,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 // Route is route object.
 type Route struct {
 	name        string
+	summary     string
 	operationID string
 	pathPattern string
 	count       int
@@ -142,6 +148,11 @@ func (r Route) Name() string {
 	return r.name
 }
 
+// Summary returns OpenAPI summary.
+func (r Route) Summary() string {
+	return r.summary
+}
+
 // OperationID returns OpenAPI operationId.
 func (r Route) OperationID() string {
 	return r.operationID
@@ -183,6 +194,11 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 		}()
 	}
 
+	elem, ok := s.cutPrefix(elem)
+	if !ok {
+		return r, false
+	}
+
 	// Static code generated router with unwrapped path search.
 	switch {
 	default:
@@ -201,6 +217,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 				switch method {
 				case "GET":
 					r.name = "GetPages"
+					r.summary = "Get all pages"
 					r.operationID = "getPages"
 					r.pathPattern = "/pages"
 					r.args = args
@@ -208,6 +225,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 					return r, true
 				case "POST":
 					r.name = "AddPage"
+					r.summary = "Add new page"
 					r.operationID = "addPage"
 					r.pathPattern = "/pages"
 					r.args = args
@@ -238,6 +256,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 					switch method {
 					case "GET":
 						r.name = "GetPage"
+						r.summary = ""
 						r.operationID = "getPage"
 						r.pathPattern = "/pages/{id}"
 						r.args = args
@@ -265,6 +284,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
 						case "GET":
 							// Leaf: GetFile
 							r.name = "GetFile"
+							r.summary = ""
 							r.operationID = "getFile"
 							r.pathPattern = "/pages/{id}/file/{file_id}"
 							r.args = args

api/openapi/oas_schemas_gen.go

@@ -140,6 +140,16 @@ const (
 	FormatHeaders    Format = "headers"
 )
 
+// AllValues returns all Format values.
+func (Format) AllValues() []Format {
+	return []Format{
+		FormatAll,
+		FormatPdf,
+		FormatSingleFile,
+		FormatHeaders,
+	}
+}
+
 // MarshalText implements encoding.TextMarshaler.
 func (s Format) MarshalText() ([]byte, error) {
 	switch s {
@@ -189,6 +199,9 @@ type GetFileOKApplicationPdf struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKApplicationPdf) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -202,6 +215,9 @@ type GetFileOKTextHTML struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKTextHTML) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -215,6 +231,9 @@ type GetFileOKTextPlain struct {
 //
 // Kept to satisfy the io.Reader interface.
 func (s GetFileOKTextPlain) Read(p []byte) (n int, err error) {
+	if s.Data == nil {
+		return 0, io.EOF
+	}
 	return s.Data.Read(p)
 }
 
@@ -324,6 +343,7 @@ type Page struct {
 	Created time.Time `json:"created"`
 	Formats []Format  `json:"formats"`
 	Status  Status    `json:"status"`
+	Meta    PageMeta  `json:"meta"`
 }
 
 // GetID returns the value of ID.
@@ -351,6 +371,11 @@ func (s *Page) GetStatus() Status {
 	return s.Status
 }
 
+// GetMeta returns the value of Meta.
+func (s *Page) GetMeta() PageMeta {
+	return s.Meta
+}
+
 // SetID sets the value of ID.
 func (s *Page) SetID(val uuid.UUID) {
 	s.ID = val
@@ -376,17 +401,59 @@ func (s *Page) SetStatus(val Status) {
 	s.Status = val
 }
 
+// SetMeta sets the value of Meta.
+func (s *Page) SetMeta(val PageMeta) {
+	s.Meta = val
+}
+
 func (*Page) addPageRes() {}
 
+type PageMeta struct {
+	Title       string    `json:"title"`
+	Description string    `json:"description"`
+	Error       OptString `json:"error"`
+}
+
+// GetTitle returns the value of Title.
+func (s *PageMeta) GetTitle() string {
+	return s.Title
+}
+
+// GetDescription returns the value of Description.
+func (s *PageMeta) GetDescription() string {
+	return s.Description
+}
+
+// GetError returns the value of Error.
+func (s *PageMeta) GetError() OptString {
+	return s.Error
+}
+
+// SetTitle sets the value of Title.
+func (s *PageMeta) SetTitle(val string) {
+	s.Title = val
+}
+
+// SetDescription sets the value of Description.
+func (s *PageMeta) SetDescription(val string) {
+	s.Description = val
+}
+
+// SetError sets the value of Error.
+func (s *PageMeta) SetError(val OptString) {
+	s.Error = val
+}
+
 // Merged schema.
 // Ref: #/components/schemas/pageWithResults
 type PageWithResults struct {
-	ID      uuid.UUID `json:"id"`
+	ID      uuid.UUID           `json:"id"`
-	URL     string    `json:"url"`
+	URL     string              `json:"url"`
-	Created time.Time `json:"created"`
+	Created time.Time           `json:"created"`
-	Formats []Format  `json:"formats"`
+	Formats []Format            `json:"formats"`
-	Status  Status    `json:"status"`
+	Status  Status              `json:"status"`
-	Results []Result  `json:"results"`
+	Meta    PageWithResultsMeta `json:"meta"`
+	Results []Result            `json:"results"`
 }
 
 // GetID returns the value of ID.
@@ -414,6 +481,11 @@ func (s *PageWithResults) GetStatus() Status {
 	return s.Status
 }
 
+// GetMeta returns the value of Meta.
+func (s *PageWithResults) GetMeta() PageWithResultsMeta {
+	return s.Meta
+}
+
 // GetResults returns the value of Results.
 func (s *PageWithResults) GetResults() []Result {
 	return s.Results
@@ -444,6 +516,11 @@ func (s *PageWithResults) SetStatus(val Status) {
 	s.Status = val
 }
 
+// SetMeta sets the value of Meta.
+func (s *PageWithResults) SetMeta(val PageWithResultsMeta) {
+	s.Meta = val
+}
+
 // SetResults sets the value of Results.
 func (s *PageWithResults) SetResults(val []Result) {
 	s.Results = val
@@ -451,6 +528,42 @@ func (s *PageWithResults) SetResults(val []Result) {
 
 func (*PageWithResults) getPageRes() {}
 
+type PageWithResultsMeta struct {
+	Title       string    `json:"title"`
+	Description string    `json:"description"`
+	Error       OptString `json:"error"`
+}
+
+// GetTitle returns the value of Title.
+func (s *PageWithResultsMeta) GetTitle() string {
+	return s.Title
+}
+
+// GetDescription returns the value of Description.
+func (s *PageWithResultsMeta) GetDescription() string {
+	return s.Description
+}
+
+// GetError returns the value of Error.
+func (s *PageWithResultsMeta) GetError() OptString {
+	return s.Error
+}
+
+// SetTitle sets the value of Title.
+func (s *PageWithResultsMeta) SetTitle(val string) {
+	s.Title = val
+}
+
+// SetDescription sets the value of Description.
+func (s *PageWithResultsMeta) SetDescription(val string) {
+	s.Description = val
+}
+
+// SetError sets the value of Error.
+func (s *PageWithResultsMeta) SetError(val OptString) {
+	s.Error = val
+}
+
 type Pages []Page
 
 // Ref: #/components/schemas/result
@@ -548,6 +661,17 @@ const (
 	StatusWithErrors Status = "with_errors"
 )
 
+// AllValues returns all Status values.
+func (Status) AllValues() []Status {
+	return []Status{
+		StatusNew,
+		StatusProcessing,
+		StatusDone,
+		StatusFailed,
+		StatusWithErrors,
+	}
+}
+
 // MarshalText implements encoding.TextMarshaler.
 func (s Status) MarshalText() ([]byte, error) {
 	switch s {

api/openapi/oas_validators_gen.go

@@ -42,6 +42,7 @@ func (s *AddPageReq) Validate() error {
 	}
 	return nil
 }
+
 func (s Format) Validate() error {
 	switch s {
 	case "all":
@@ -103,6 +104,7 @@ func (s *Page) Validate() error {
 	}
 	return nil
 }
+
 func (s *PageWithResults) Validate() error {
 	var failures []validate.FieldError
 	if err := func() error {
@@ -177,12 +179,14 @@ func (s *PageWithResults) Validate() error {
 	}
 	return nil
 }
+
 func (s Pages) Validate() error {
-	if s == nil {
+	alias := ([]Page)(s)
+	if alias == nil {
 		return errors.New("nil is invalid value")
 	}
 	var failures []validate.FieldError
-	for i, elem := range s {
+	for i, elem := range alias {
 		if err := func() error {
 			if err := elem.Validate(); err != nil {
 				return err
@@ -200,6 +204,7 @@ func (s Pages) Validate() error {
 	}
 	return nil
 }
+
 func (s *Result) Validate() error {
 	var failures []validate.FieldError
 	if err := func() error {
@@ -229,6 +234,7 @@ func (s *Result) Validate() error {
 	}
 	return nil
 }
+
 func (s Status) Validate() error {
 	switch s {
 	case "new":

application/application.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
@@ -15,6 +16,8 @@ import (
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
+	"github.com/derfenix/webarchive/adapters/repository"
+
 	"github.com/derfenix/webarchive/adapters/processors"
 	badgerRepo "github.com/derfenix/webarchive/adapters/repository/badger"
 	"github.com/derfenix/webarchive/api/openapi"
@@ -29,7 +32,7 @@ func NewApplication(cfg config.Config) (Application, error) {
 		return Application{}, fmt.Errorf("new logger: %w", err)
 	}
 
-	db, err := badgerRepo.NewBadger(cfg.DB.Path, log.Named("db"))
+	db, err := repository.NewBadger(cfg.DB.Path, log.Named("db"))
 	if err != nil {
 		return Application{}, fmt.Errorf("new badger: %w", err)
 	}
@@ -39,7 +42,7 @@ func NewApplication(cfg config.Config) (Application, error) {
 		return Application{}, fmt.Errorf("new page repo: %w", err)
 	}
 
-	processor, err := processors.NewProcessors(cfg)
+	processor, err := processors.NewProcessors(cfg, log.Named("processor"))
 	if err != nil {
 		return Application{}, fmt.Errorf("new processors: %w", err)
 	}
@@ -48,7 +51,8 @@ func NewApplication(cfg config.Config) (Application, error) {
 	worker := entity.NewWorker(workerCh, pageRepo, processor, log.Named("worker"))
 
 	server, err := openapi.NewServer(
-		rest.NewService(pageRepo, workerCh),
+		rest.NewService(pageRepo, workerCh, processor),
+		openapi.WithPathPrefix("/api/v1"),
 		openapi.WithMiddleware(
 			func(r middleware.Request, next middleware.Next) (middleware.Response, error) {
 				start := time.Now()
@@ -73,9 +77,25 @@ func NewApplication(cfg config.Config) (Application, error) {
 		return Application{}, fmt.Errorf("new rest server: %w", err)
 	}
 
+	var httpHandler http.Handler = server
+
+	if cfg.UI.Enabled {
+		ui := rest.NewUI(cfg.UI)
+
+		httpHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if strings.HasPrefix(r.URL.Path, "/api/") {
+				server.ServeHTTP(w, r)
+
+				return
+			}
+
+			ui.ServeHTTP(w, r)
+		})
+	}
+
 	httpServer := http.Server{
 		Addr:              cfg.API.Address,
-		Handler:           server,
+		Handler:           httpHandler,
 		ReadTimeout:       time.Second * 15,
 		ReadHeaderTimeout: time.Second * 5,
 		IdleTimeout:       time.Second * 30,
@@ -155,7 +175,7 @@ func (a *Application) Stop() error {
 		errs = multierr.Append(errs, fmt.Errorf("sync db: %w", err))
 	}
 
-	if err := badgerRepo.Backup(a.db, badgerRepo.BackupStop); err != nil {
+	if err := repository.Backup(a.db, repository.BackupStop); err != nil {
 		errs = multierr.Append(errs, fmt.Errorf("backup on stop: %w", err))
 	}
 
@@ -171,6 +191,7 @@ func newLogger(cfg config.Logging) (*zap.Logger, error) {
 	logCfg.EncoderConfig.EncodeTime = zapcore.RFC3339TimeEncoder
 	logCfg.EncoderConfig.EncodeDuration = zapcore.NanosDurationEncoder
 	logCfg.DisableCaller = true
+	logCfg.DisableStacktrace = true
 
 	logCfg.Level = zap.NewAtomicLevelAt(zapcore.InfoLevel)
 	if cfg.Debug {

config/config.go

@@ -28,6 +28,7 @@ type Config struct {
 	DB      DB      `env:",prefix=DB_"`
 	Logging Logging `env:",prefix=LOGGING_"`
 	API     API     `env:",prefix=API_"`
+	UI      UI      `env:",prefix=UI_"`
 	PDF     PDF     `env:",prefix=PDF_"`
 }
 
@@ -36,8 +37,8 @@ type PDF struct {
 	Grayscale  bool    `env:"GRAYSCALE,default=false"`
 	MediaPrint bool    `env:"MEDIA_PRINT,default=true"`
 	Zoom       float64 `env:"ZOOM,default=1"`
-	Viewport   string  `env:"VIEWPORT,default=1920x1080"`
+	Viewport   string  `env:"VIEWPORT,default=1280x720"`
-	DPI        uint    `env:"DPI,default=300"`
+	DPI        uint    `env:"DPI,default=150"`
 	Filename   string  `env:"FILENAME,default=page.pdf"`
 }
 
@@ -45,6 +46,12 @@ type API struct {
 	Address string `env:"ADDRESS,default=0.0.0.0:5001"`
 }
 
+type UI struct {
+	Enabled bool   `env:"ENABLED,default=true"`
+	Prefix  string `env:"PREFIX,default=/"`
+	Theme   string `env:"THEME,default=basic"`
+}
+
 type DB struct {
 	Path string `env:"PATH,default=./db"`
 }

docker-compose.yaml

@@ -2,14 +2,15 @@ version: "3"
 
 services:
   webarchive:
-    build:
+    image: ghcr.io/derfenix/webarchive:latest
-      dockerfile: ./Dockerfile
+#    build:
-      context: .
+#      dockerfile: ./Dockerfile
+#      context: .
     environment:
-      LOGGING_DEBUG: true
+      LOGGING_DEBUG: "true"
-      API_ADDRESS: 0.0.0.0:5001
+      API_ADDRESS: "0.0.0.0:5001"
-      PDF_DPI: 300
+      PDF_DPI: "300"
-      DB_PATH: /db
+      DB_PATH: "/db"
     volumes:
       - ./db:/db
     ports:

entity/cache.go

@@ -0,0 +1,42 @@
+package entity
+
+import (
+	"bytes"
+	"io"
+	"sync"
+)
+
+func NewCache() *Cache {
+	return &Cache{data: make([]byte, 0, 1024*512)}
+}
+
+type Cache struct {
+	mu   sync.RWMutex
+	data []byte
+}
+
+func (c *Cache) Write(p []byte) (n int, err error) {
+	c.mu.Lock()
+	c.data = append(c.data, p...)
+	c.mu.Unlock()
+
+	return len(p), nil
+}
+
+func (c *Cache) Get() []byte {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.data
+}
+
+func (c *Cache) Reader() io.Reader {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if len(c.data) == 0 {
+		return nil
+	}
+
+	return bytes.NewBuffer(c.data)
+}

entity/page.go

@@ -3,6 +3,7 @@ package entity
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -10,7 +11,8 @@ import (
 )
 
 type Processor interface {
-	Process(ctx context.Context, format Format, url string) Result
+	Process(ctx context.Context, format Format, url string, cache *Cache) Result
+	GetMeta(ctx context.Context, url string, cache *Cache) (Meta, error)
 }
 
 type Format uint8
@@ -37,55 +39,82 @@ const (
 	StatusWithErrors
 )
 
-func NewPage(url string, description string, formats ...Format) *Page {
+type Meta struct {
-	return &Page{
+	Title       string
-		ID:          uuid.New(),
+	Description string
-		URL:         url,
+	Encoding    string
-		Description: description,
+	Error       string
-		Formats:     formats,
-		Created:     time.Now(),
-		Version:     1,
-	}
 }
 
-type Page struct {
+type PageBase struct {
 	ID          uuid.UUID
 	URL         string
 	Description string
 	Created     time.Time
 	Formats     []Format
-	Results     Results
 	Version     uint16
 	Status      Status
+	Meta        Meta
+}
+
+func NewPage(url string, description string, formats ...Format) *Page {
+	return &Page{
+		PageBase: PageBase{
+			ID:          uuid.New(),
+			URL:         url,
+			Description: description,
+			Formats:     formats,
+			Created:     time.Now(),
+			Version:     1,
+		},
+		cache: NewCache(),
+	}
+}
+
+type Page struct {
+	PageBase
+	Results ResultsRO
+	cache   *Cache
 }
 
 func (p *Page) SetProcessing() {
 	p.Status = StatusProcessing
 }
 
+func (p *Page) Prepare(ctx context.Context, processor Processor) {
+	meta, err := processor.GetMeta(ctx, p.URL, p.cache)
+	if err != nil {
+		p.Meta.Error = err.Error()
+	} else {
+		p.Meta = meta
+	}
+}
+
 func (p *Page) Process(ctx context.Context, processor Processor) {
 	innerWG := sync.WaitGroup{}
 	innerWG.Add(len(p.Formats))
 
+	results := Results{}
+
 	for _, format := range p.Formats {
 		go func(format Format) {
 			defer innerWG.Done()
 
 			defer func() {
 				if err := recover(); err != nil {
-					p.Results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v", err)})
+					results.Add(Result{Format: format, Err: fmt.Errorf("recovered from panic: %v (%s)", err, string(debug.Stack()))})
 				}
 			}()
 
-			result := processor.Process(ctx, format, p.URL)
+			result := processor.Process(ctx, format, p.URL, p.cache)
-			p.Results.Add(result)
+			results.Add(result)
 		}(format)
 	}
 
 	innerWG.Wait()
 
 	var hasResultWithOutErrors bool
-	for _, result := range p.Results.Results() {
+	for _, result := range results.Results() {
 		if result.Err != nil {
 			p.Status = StatusWithErrors
 		} else {
@@ -100,4 +129,6 @@ func (p *Page) Process(ctx context.Context, processor Processor) {
 	if p.Status == StatusProcessing {
 		p.Status = StatusDone
 	}
+
+	p.Results = results.RO()
 }

entity/result.go

@@ -6,6 +6,8 @@ import (
 	"github.com/vmihailenco/msgpack/v5"
 )
 
+type ResultsRO []Result
+
 type Result struct {
 	Format Format
 	Err    error
@@ -17,6 +19,13 @@ type Results struct {
 	results []Result
 }
 
+func (r *Results) RO() ResultsRO {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.results
+}
+
 func (r *Results) MarshalMsgpack() ([]byte, error) {
 	return msgpack.Marshal(r.results)
 }

entity/worker.go

@@ -9,6 +9,7 @@ import (
 
 type Pages interface {
 	Save(ctx context.Context, page *Page) error
+	ListUnprocessed(ctx context.Context) ([]Page, error)
 }
 
 func NewWorker(ch chan *Page, pages Pages, processor Processor, log *zap.Logger) *Worker {
@@ -27,6 +28,20 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 	w.log.Info("starting")
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		unprocessed, err := w.pages.ListUnprocessed(ctx)
+		if err != nil {
+			w.log.Error("failed to get unprocessed pages", zap.Error(err))
+		} else {
+			for i := range unprocessed {
+				w.ch <- &unprocessed[i]
+			}
+		}
+	}()
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -51,6 +66,16 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
 func (w *Worker) do(ctx context.Context, wg *sync.WaitGroup, page *Page, log *zap.Logger) {
 	defer wg.Done()
 
+	page.SetProcessing()
+	if err := w.pages.Save(ctx, page); err != nil {
+		w.log.Error(
+			"failed to save processing page",
+			zap.String("page_id", page.ID.String()),
+			zap.String("page_url", page.URL),
+			zap.Error(err),
+		)
+	}
+
 	page.Process(ctx, w.processor)
 
 	log.Debug("page processed")

go.mod

@@ -5,33 +5,35 @@ go 1.19
 require (
 	github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0
 	github.com/dgraph-io/badger/v4 v4.0.1
+	github.com/disintegration/imaging v1.6.2
 	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/go-faster/errors v0.6.1
-	github.com/go-faster/jx v1.0.0
+	github.com/go-faster/jx v1.1.0
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.4.0
-	github.com/ogen-go/ogen v0.60.1
+	github.com/minio/minio-go/v7 v7.0.52
+	github.com/ogen-go/ogen v0.77.0
 	github.com/sethvargo/go-envconfig v0.9.0
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
 	github.com/vmihailenco/msgpack/v5 v5.3.5
-	go.opentelemetry.io/otel v1.14.0
+	go.opentelemetry.io/otel v1.19.0
-	go.opentelemetry.io/otel/metric v0.37.0
+	go.opentelemetry.io/otel/metric v1.19.0
-	go.opentelemetry.io/otel/trace v1.14.0
+	go.opentelemetry.io/otel/trace v1.19.0
-	go.uber.org/multierr v1.10.0
+	go.uber.org/multierr v1.11.0
-	go.uber.org/zap v1.24.0
+	go.uber.org/zap v1.26.0
+	golang.org/x/net v0.23.0
 )
 
 require (
-	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
-	github.com/dlclark/regexp2 v1.8.1 // indirect
+	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
-	github.com/go-faster/yamlx v0.4.1 // indirect
+	github.com/go-faster/yaml v0.4.6 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.1.1 // indirect
@@ -39,21 +41,30 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/flatbuffers v23.3.3+incompatible // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.16.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/rs/xid v1.4.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
+	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
-	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/image v0.18.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -4,8 +4,6 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0 h1:DNrExYwvyyI404SxdUCCANAj9TwnGjRfa3cYFMNY1AU=
 github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0/go.mod h1:SQq4xfIdvf6WYKSDxAJc+xOJdolt+/bc1jnQKMtPMvQ=
-github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
-github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -23,8 +21,10 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa
 github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0=
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
-github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -40,13 +40,13 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI=
 github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY=
-github.com/go-faster/jx v1.0.0 h1:HE+ms2e6ZGkZ6u13t8u+onBinrPvIPI+0hWXGELm74g=
+github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg=
-github.com/go-faster/jx v1.0.0/go.mod h1:zm8SlkwK+H0TYNKYtVJ/7cWFS7soJBQWhcPctKyYL/4=
+github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg=
-github.com/go-faster/yamlx v0.4.1 h1:00RQkZopoLDF1SgBDJVHuN6epTOK7T0TkN427vbvEBk=
+github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
-github.com/go-faster/yamlx v0.4.1/go.mod h1:QXr/i3Z00jRhskgyWkoGsEdseebd/ZbZEpGS6DJv8oo=
+github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
-github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
-github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -82,32 +82,54 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
 github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
+github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/ogen-go/ogen v0.60.1 h1:yOt0i6NcH7jM3rBi9nnv5VsGUQRw4ACUMsiJojnqrAM=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
-github.com/ogen-go/ogen v0.60.1/go.mod h1:tcwLpHe4vyk9xtbTMe3yu3Qtcbz8VjrpBz9LzsdwWvQ=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps=
+github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/ogen-go/ogen v0.77.0 h1:yREPDg3cDuXkDyp7FPXdPEUz+azPZFUGKmYer8fJpmM=
+github.com/ogen-go/ogen v0.77.0/go.mod h1:/bl+MubIppovr7F1fKAaDxzFF+oF2EiMtyVylyqDtQ8=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE=
 github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0=
+github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
+github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -116,11 +138,12 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
 github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
@@ -129,25 +152,28 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
-go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
-go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
-go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
-go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
-go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
-go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
+go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
-go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
-go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
+go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
-go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
-go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
-go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
-golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
+golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -162,28 +188,31 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -217,10 +246,12 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

ports/rest/converter.go

@@ -2,6 +2,7 @@ package rest
 
 import (
 	"fmt"
+	"html"
 
 	"github.com/derfenix/webarchive/api/openapi"
 	"github.com/derfenix/webarchive/entity"
@@ -22,11 +23,16 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults {
 			return res
 		}(),
 		Status: StatusToRest(page.Status),
+		Meta: openapi.PageWithResultsMeta{
+			Title:       html.EscapeString(page.Meta.Title),
+			Description: html.EscapeString(page.Meta.Description),
+			Error:       openapi.NewOptString(page.Meta.Error),
+		},
 		Results: func() []openapi.Result {
-			results := make([]openapi.Result, len(page.Results.Results()))
+			results := make([]openapi.Result, len(page.Results))
 
 			for i := range results {
-				result := &(page.Results.Results())[i]
+				result := &page.Results[i]
 
 				errText := openapi.OptString{}
 				if result.Err != nil {
@@ -60,11 +66,39 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults {
 	}
 }
 
+func BasePageToRest(page *entity.PageBase) openapi.Page {
+	return openapi.Page{
+		ID:      page.ID,
+		URL:     page.URL,
+		Created: page.Created,
+		Meta: openapi.PageMeta{
+			Title:       html.EscapeString(page.Meta.Title),
+			Description: html.EscapeString(page.Meta.Description),
+			Error:       openapi.NewOptString(page.Meta.Error),
+		},
+		Formats: func() []openapi.Format {
+			res := make([]openapi.Format, len(page.Formats))
+
+			for i, format := range page.Formats {
+				res[i] = FormatToRest(format)
+			}
+
+			return res
+		}(),
+		Status: StatusToRest(page.Status),
+	}
+}
+
 func PageToRest(page *entity.Page) openapi.Page {
 	return openapi.Page{
 		ID:      page.ID,
 		URL:     page.URL,
 		Created: page.Created,
+		Meta: openapi.PageMeta{
+			Title:       html.EscapeString(page.Meta.Title),
+			Description: html.EscapeString(page.Meta.Description),
+			Error:       openapi.NewOptString(page.Meta.Error),
+		},
 		Formats: func() []openapi.Format {
 			res := make([]openapi.Format, len(page.Formats))
 

ports/rest/service.go

@@ -20,14 +20,19 @@ type Pages interface {
 	GetFile(ctx context.Context, pageID, fileID uuid.UUID) (*entity.File, error)
 }
 
-func NewService(sites Pages, ch chan *entity.Page) *Service {
+func NewService(pages Pages, ch chan *entity.Page, processor entity.Processor) *Service {
-	return &Service{pages: sites, ch: ch}
+	return &Service{
+		pages:     pages,
+		ch:        ch,
+		processor: processor,
+	}
 }
 
 type Service struct {
 	openapi.UnimplementedHandler
-	pages Pages
+	processor entity.Processor
-	ch    chan *entity.Page
+	pages     Pages
+	ch        chan *entity.Page
 }
 
 func (s *Service) GetPage(ctx context.Context, params openapi.GetPageParams) (openapi.GetPageRes, error) {
@@ -76,13 +81,14 @@ func (s *Service) AddPage(ctx context.Context, req openapi.OptAddPageReq, params
 	}
 
 	page := entity.NewPage(url, description, domainFormats...)
-	page.Status = entity.StatusProcessing
+	page.Status = entity.StatusNew
+	page.Prepare(ctx, s.processor)
 
 	if err := s.pages.Save(ctx, page); err != nil {
 		return nil, fmt.Errorf("save page: %w", err)
 	}
 
-	res := PageToRest(page)
+	res := BasePageToRest(&page.PageBase)
 
 	s.ch <- page
 

ports/rest/ui.go

@@ -0,0 +1,41 @@
+package rest
+
+import (
+	"io/fs"
+	"net/http"
+	"strings"
+
+	"github.com/derfenix/webarchive/config"
+	"github.com/derfenix/webarchive/ui"
+)
+
+func NewUI(cfg config.UI) *UI {
+	return &UI{
+		prefix: cfg.Prefix,
+		theme:  cfg.Theme,
+	}
+}
+
+type UI struct {
+	prefix string
+	theme  string
+}
+
+func (u *UI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	serveRoot, err := fs.Sub(ui.StaticFiles, u.theme)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	if strings.HasPrefix(r.URL.Path, u.prefix) {
+		r.URL.Path = "/" + strings.TrimPrefix(r.URL.Path, u.prefix)
+	}
+	if !strings.HasPrefix(r.URL.Path, "/static") {
+		r.URL.Path = "/"
+	}
+
+	r.URL.Path = strings.TrimPrefix(r.URL.Path, "/static")
+
+	http.FileServer(http.FS(serveRoot)).ServeHTTP(w, r)
+}

ui/basic/index.html

@@ -0,0 +1,47 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>WebArchive</title>
+    <link rel="stylesheet" href="/static/style.css">
+    <script src="/static/lib.js"></script>
+    <script src="/static/main.js"></script>
+</head>
+<body>
+<template id="pages_tmpl">
+    <div class="page_item">
+        <a class="url link"><span class="title"></span><span class="status"></span></a>
+        <div class="description"></div>
+        <div class="created"></div>
+        <hr>
+    </div>
+</template>
+
+<template id="page_tmpl">
+    <a onclick="history.back()" class="link">Back</a>
+    <div class="page">
+        <h2 id="page_title"></h2>
+        <h3 id="page_description"></h3>
+        <h5 id="page_url" class="link" onclick="window.open(this.innerHTML, '_blank')"></h5>
+        <h4>Results</h4>
+        <div id="results"></div>
+    </div>
+</template>
+
+<template id="result_tmpl">
+    <div class="result_item">
+        <span class="format"></span>
+        <span class="result_link link"></span>
+    </div>
+</template>
+
+<h1 id="site_title"></h1>
+
+<div id="data">
+    None
+</div>
+</body>
+</html>
+

ui/basic/main.js

@@ -0,0 +1,90 @@
+function index() {
+  $.ajax({
+    url: "/api/v1/pages", success: function (data, status, xhr) {
+      if (status !== "success") {
+        gotError(status);
+
+        return;
+      }
+
+      let elem = document.getElementById("data");
+      elem.innerHTML = "";
+      // elem.attachShadow({mode: 'open'});
+
+      data.forEach(function (v) {
+        let page_elem = pages_tmpl.content.cloneNode(true);
+        $(page_elem).find(".url").attr("onclick", "goToPage('" + v.id + "');");
+        $(page_elem).find(".status").addClass(v.status);
+        $(page_elem).find(".status").attr("title", v.status);
+        $(page_elem).find(".created").html(v.created);
+        $(page_elem).find(".title").html(v.meta.title);
+        $(page_elem).find(".description").html(v.meta.description);
+        elem.append(page_elem); // (*)
+      })
+    }
+  })
+}
+
+function goToPage(id) {
+  history.pushState({"page": id}, null, id);
+  page(id);
+}
+
+function page(id) {
+  $.ajax({
+    url: "/api/v1/pages/" + id, success: function (data, status, xhr) {
+      if (status !== "success") {
+        gotError(status);
+
+        return;
+      }
+
+      let elem = document.getElementById("data");
+      elem.innerHTML = "";
+      let page_elem = page_tmpl.content.cloneNode(true);
+      $(page_elem).find("#page_title").html(data.meta.title);
+      $(page_elem).find("#page_description").html(data.meta.description);
+      $(page_elem).find("#page_url").html(data.url);
+
+      data.results.forEach(function (result) {
+        let result_elem = result_tmpl.content.cloneNode(true);
+        $(result_elem).find(".format").html(result.format);
+        if (result.error !== "" && result.error !== undefined) {
+          $(result_elem).find(".format").addClass("error");
+          $(result_elem).find(".result_link").html("⚠");
+          $(result_elem).find(".result_link").attr("title", result.error);
+        } else {
+          result.files.forEach(function (file) {
+            $(result_elem).find(".result_link").attr("onclick", "window.open('/api/v1/pages/" + data.id + "/file/" + file.id + "', '_blank');");
+            $(result_elem).find(".result_link").html(file.name);
+          })
+        }
+
+        $(page_elem).find("#results").append(result_elem);
+      })
+
+      elem.append(page_elem); // (*)
+    }
+  })
+}
+
+function gotError(err) {
+  console.log(err);
+}
+
+document.addEventListener("DOMContentLoaded", function () {
+  $("#site_title").html("WebArchive " + window.location.hostname);
+  document.title = "WebArchive " + window.location.hostname;
+  if (window.location.pathname.endsWith("/")) {
+    index();
+  } else {
+    page(window.location.pathname.slice(1));
+  }
+});
+window.addEventListener('popstate', function (event) {
+  if (event.state === null) {
+    index();
+  } else {
+    page(event.state.page);
+  }
+});

ui/embed.go

@@ -0,0 +1,8 @@
+package ui
+
+import (
+	"embed"
+)
+
+//go:embed */*.html */*.css */*.js
+var StaticFiles embed.FS