fenix/webarchive
1
0
Fork 0
mirror of https://github.com/derfenix/webarchive.git synced 2026-03-11 12:41:54 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update deps, refactoring

Browse Source
This commit is contained in:
derFenix
2023-11-01 12:53:43 +03:00
parent 7cd4d4097a
commit e652abb4bd
24 changed files with 540 additions and 238 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/release.yaml vendored
View File

@@ -1,6 +1,7 @@
---
name: release
on:
"on":
push:
tags:
- 'v*'

2
adapters/repository/badger/db.go → adapters/repository/badger.go
View File

@@ -1,4 +1,4 @@
package badger
package repository
import (
"errors"

35
adapters/repository/badger/page.go
View File

@@ -8,6 +8,8 @@ import (
"github.com/dgraph-io/badger/v4"
"github.com/google/uuid"
"github.com/derfenix/webarchive/adapters/repository"
"github.com/derfenix/webarchive/entity"
)
@@ -24,7 +26,9 @@ type Page struct {
}
func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
page := entity.Page{ID: pageID}
page := entity.Page{}
page.ID = pageID
var file *entity.File
err := p.db.View(func(txn *badger.Txn) error {
@@ -66,7 +70,7 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil
func (p *Page) Save(_ context.Context, page *entity.Page) error {
if p.db.IsClosed() {
return ErrDBClosed
return repository.ErrDBClosed
}
marshaled, err := marshal(page)
@@ -88,16 +92,17 @@ func (p *Page) Save(_ context.Context, page *entity.Page) error {
}
func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
site := entity.Page{ID: id}
page := entity.Page{}
page.ID = id
err := p.db.View(func(txn *badger.Txn) error {
data, err := txn.Get(p.key(&site))
data, err := txn.Get(p.key(&page))
if err != nil {
return fmt.Errorf("get data: %w", err)
}
err = data.Value(func(val []byte) error {
if err := unmarshal(val, &site); err != nil {
if err := unmarshal(val, &page); err != nil {
return fmt.Errorf("unmarshal data: %w", err)
}
@@ -113,7 +118,7 @@ func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) {
return nil, fmt.Errorf("view: %w", err)
}
return &site, nil
return &page, nil
}
func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
@@ -143,16 +148,7 @@ func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) {
return fmt.Errorf("get item: %w", err)
}
pages = append(pages, &entity.Page{
ID: page.ID,
URL: page.URL,
Description: page.Description,
Created: page.Created,
Formats: page.Formats,
Version: page.Version,
Status: page.Status,
Meta: page.Meta,
})
pages = append(pages, &page)
}
return nil
@@ -196,12 +192,11 @@ func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
return fmt.Errorf("get item: %w", err)
}
if page.Status != entity.StatusProcessing {
continue
if page.Status == entity.StatusNew {
//goland:noinspection GoVetCopyLock
pages = append(pages, page) //nolint:govet // didn't touch the lock here
}
//goland:noinspection GoVetCopyLock
pages = append(pages, page) //nolint:govet // didn't touch the lock here
}
return nil

3
adapters/repository/badger/page_test.go
View File

@@ -6,6 +6,7 @@ import (
"testing"
"time"
"github.com/derfenix/webarchive/adapters/repository"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
@@ -31,7 +32,7 @@ func TestSite(t *testing.T) {
log := zaptest.NewLogger(t)
db, err := NewBadger(tempDir, log.Named("db"))
db, err := repository.NewBadger(tempDir, log.Named("db"))
require.NoError(t, err)
siteRepo, err := NewPage(db)

13
adapters/repository/badgers3/marshal.go Normal file
View File

@@ -0,0 +1,13 @@
package badgers3
import (
"github.com/vmihailenco/msgpack/v5"
)
func marshal(v interface{}) ([]byte, error) {
return msgpack.Marshal(v)
}
func unmarshal(b []byte, v interface{}) error {
return msgpack.Unmarshal(b, v)
}

119
adapters/repository/badgers3/page.go Normal file
View File

@@ -0,0 +1,119 @@
package badgers3
import (
"bytes"
"context"
"errors"
"fmt"
"time"
"github.com/derfenix/webarchive/adapters/repository"
"github.com/derfenix/webarchive/entity"
"github.com/dgraph-io/badger/v4"
"github.com/google/uuid"
"github.com/minio/minio-go/v7"
)
func NewPage(db *badger.DB, s3 *minio.Client, bucketName string) (*Page, error) {
return &Page{
db: db,
s3: s3,
prefix: []byte("pages3:"),
bucketName: bucketName,
}, nil
}
type Page struct {
db *badger.DB
s3 *minio.Client
prefix []byte
bucketName string
}
func (p *Page) ListAll(ctx context.Context) ([]*entity.PageBase, error) {
// TODO implement me
panic("implement me")
}
func (p *Page) Get(ctx context.Context, id uuid.UUID) (*entity.Page, error) {
// TODO implement me
panic("implement me")
}
func (p *Page) GetFile(ctx context.Context, pageID, fileID uuid.UUID) (*entity.File, error) {
// TODO implement me
panic("implement me")
}
func (p *Page) Save(ctx context.Context, page *entity.Page) error {
if p.db.IsClosed() {
return repository.ErrDBClosed
}
marshaled, err := marshal(page.PageBase)
if err != nil {
return fmt.Errorf("marshal data: %w", err)
}
if err := p.db.Update(func(txn *badger.Txn) error {
if err := txn.Set(p.key(page), marshaled); err != nil {
return fmt.Errorf("put data: %w", err)
}
return nil
}); err != nil {
return fmt.Errorf("update db: %w", err)
}
snowball := make(chan minio.SnowballObject, 1)
go func() {
defer close(snowball)
for _, result := range page.Results {
for _, file := range result.Files {
for {
if ctx.Err() != nil {
return
}
if len(snowball) < cap(snowball) {
break
}
}
snowball <- minio.SnowballObject{
Key: file.ID.String(),
Size: int64(len(file.Data)),
ModTime: time.Now(),
Content: bytes.NewReader(file.Data),
}
}
}
}()
if err = p.s3.PutObjectsSnowball(ctx, p.bucketName, minio.SnowballOptions{Compress: true}, snowball); err != nil {
if dErr := p.db.Update(func(txn *badger.Txn) error {
if err := txn.Delete(p.key(page)); err != nil {
return fmt.Errorf("put data: %w", err)
}
return nil
}); dErr != nil {
err = errors.Join(err, dErr)
}
return fmt.Errorf("store files to s3: %w", err)
}
return nil
}
func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) {
// TODO implement me
panic("implement me")
}
func (p *Page) key(site *entity.Page) []byte {
return append(p.prefix, []byte(site.ID.String())...)
}

2
api/gen.go
View File

@@ -1,3 +1,3 @@
package api
//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.60.1 --target ./openapi -package openapi --clean openapi.yaml
//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.77.0 --target ./openapi -package openapi --clean openapi.yaml

21
api/openapi/oas_cfg_gen.go
View File

@@ -7,7 +7,6 @@ import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/metric/instrument"
"go.opentelemetry.io/otel/trace"
ht "github.com/ogen-go/ogen/http"
@@ -40,7 +39,7 @@ func (cfg *otelConfig) initOTEL() {
cfg.TracerProvider = otel.GetTracerProvider()
}
if cfg.MeterProvider == nil {
cfg.MeterProvider = metric.NewNoopMeterProvider()
cfg.MeterProvider = otel.GetMeterProvider()
}
cfg.Tracer = cfg.TracerProvider.Tracer(otelogen.Name,
trace.WithInstrumentationVersion(otelogen.SemVersion()),
@@ -99,9 +98,9 @@ func newServerConfig(opts ...ServerOption) serverConfig {
type baseServer struct {
cfg serverConfig
requests instrument.Int64Counter
errors instrument.Int64Counter
duration instrument.Int64Histogram
requests metric.Int64Counter
errors metric.Int64Counter
duration metric.Float64Histogram
}
func (s baseServer) notFound(w http.ResponseWriter, r *http.Request) {
@@ -120,7 +119,7 @@ func (cfg serverConfig) baseServer() (s baseServer, err error) {
if s.errors, err = s.cfg.Meter.Int64Counter(otelogen.ServerErrorsCount); err != nil {
return s, err
}
if s.duration, err = s.cfg.Meter.Int64Histogram(otelogen.ServerDuration); err != nil {
if s.duration, err = s.cfg.Meter.Float64Histogram(otelogen.ServerDuration); err != nil {
return s, err
}
return s, nil
@@ -162,9 +161,9 @@ func newClientConfig(opts ...ClientOption) clientConfig {
type baseClient struct {
cfg clientConfig
requests instrument.Int64Counter
errors instrument.Int64Counter
duration instrument.Int64Histogram
requests metric.Int64Counter
errors metric.Int64Counter
duration metric.Float64Histogram
}
func (cfg clientConfig) baseClient() (c baseClient, err error) {
@@ -175,7 +174,7 @@ func (cfg clientConfig) baseClient() (c baseClient, err error) {
if c.errors, err = c.cfg.Meter.Int64Counter(otelogen.ClientErrorsCount); err != nil {
return c, err
}
if c.duration, err = c.cfg.Meter.Int64Histogram(otelogen.ClientDuration); err != nil {
if c.duration, err = c.cfg.Meter.Float64Histogram(otelogen.ClientDuration); err != nil {
return c, err
}
return c, nil
@@ -200,7 +199,7 @@ func WithTracerProvider(provider trace.TracerProvider) Option {
// WithMeterProvider specifies a meter provider to use for creating a meter.
//
// If none is specified, the metric.NewNoopMeterProvider is used.
// If none is specified, the otel.GetMeterProvider() is used.
func WithMeterProvider(provider metric.MeterProvider) Option {
return otelOptionFunc(func(cfg *otelConfig) {
if provider != nil {

82
api/openapi/oas_client_gen.go
View File

@@ -11,6 +11,8 @@ import (
"github.com/go-faster/errors"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
"go.opentelemetry.io/otel/trace"
"github.com/ogen-go/ogen/conv"
@@ -19,6 +21,34 @@ import (
"github.com/ogen-go/ogen/uri"
)
// Invoker invokes operations described by OpenAPI v3 specification.
type Invoker interface {
// AddPage invokes addPage operation.
//
// Add new page.
//
// POST /pages
AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error)
// GetFile invokes getFile operation.
//
// Get file content.
//
// GET /pages/{id}/file/{file_id}
GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error)
// GetPage invokes getPage operation.
//
// Get page details.
//
// GET /pages/{id}
GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error)
// GetPages invokes getPages operation.
//
// Get all pages.
//
// GET /pages
GetPages(ctx context.Context) (Pages, error)
}
// Client implements OAS client.
type Client struct {
serverURL *url.URL
@@ -78,19 +108,20 @@ func (c *Client) requestURL(ctx context.Context) *url.URL {
// POST /pages
func (c *Client) AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error) {
res, err := c.sendAddPage(ctx, request, params)
_ = res
return res, err
}
func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (res AddPageRes, err error) {
otelAttrs := []attribute.KeyValue{
otelogen.OperationID("addPage"),
semconv.HTTPMethodKey.String("POST"),
semconv.HTTPRouteKey.String("/pages"),
}
// Validate request before sending.
if err := func() error {
if request.Set {
if value, ok := request.Get(); ok {
if err := func() error {
if err := request.Value.Validate(); err != nil {
if err := value.Validate(); err != nil {
return err
}
return nil
@@ -106,12 +137,13 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
// Run stopwatch.
startTime := time.Now()
defer func() {
// Use floating point division here for higher precision (instead of Millisecond method).
elapsedDuration := time.Since(startTime)
c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
c.requests.Add(ctx, 1, otelAttrs...)
c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
// Start a span for this request.
ctx, span := c.cfg.Tracer.Start(ctx, "AddPage",
@@ -124,7 +156,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
c.errors.Add(ctx, 1, otelAttrs...)
c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
span.End()
}()
@@ -197,7 +229,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
u.RawQuery = q.Values().Encode()
stage = "EncodeRequest"
r, err := ht.NewRequest(ctx, "POST", u, nil)
r, err := ht.NewRequest(ctx, "POST", u)
if err != nil {
return res, errors.Wrap(err, "create request")
}
@@ -228,24 +260,26 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params
// GET /pages/{id}/file/{file_id}
func (c *Client) GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error) {
res, err := c.sendGetFile(ctx, params)
_ = res
return res, err
}
func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res GetFileRes, err error) {
otelAttrs := []attribute.KeyValue{
otelogen.OperationID("getFile"),
semconv.HTTPMethodKey.String("GET"),
semconv.HTTPRouteKey.String("/pages/{id}/file/{file_id}"),
}
// Run stopwatch.
startTime := time.Now()
defer func() {
// Use floating point division here for higher precision (instead of Millisecond method).
elapsedDuration := time.Since(startTime)
c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
c.requests.Add(ctx, 1, otelAttrs...)
c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
// Start a span for this request.
ctx, span := c.cfg.Tracer.Start(ctx, "GetFile",
@@ -258,7 +292,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
c.errors.Add(ctx, 1, otelAttrs...)
c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
span.End()
}()
@@ -307,7 +341,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
uri.AddPathParts(u, pathParts[:]...)
stage = "EncodeRequest"
r, err := ht.NewRequest(ctx, "GET", u, nil)
r, err := ht.NewRequest(ctx, "GET", u)
if err != nil {
return res, errors.Wrap(err, "create request")
}
@@ -335,24 +369,26 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get
// GET /pages/{id}
func (c *Client) GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error) {
res, err := c.sendGetPage(ctx, params)
_ = res
return res, err
}
func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res GetPageRes, err error) {
otelAttrs := []attribute.KeyValue{
otelogen.OperationID("getPage"),
semconv.HTTPMethodKey.String("GET"),
semconv.HTTPRouteKey.String("/pages/{id}"),
}
// Run stopwatch.
startTime := time.Now()
defer func() {
// Use floating point division here for higher precision (instead of Millisecond method).
elapsedDuration := time.Since(startTime)
c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
c.requests.Add(ctx, 1, otelAttrs...)
c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
// Start a span for this request.
ctx, span := c.cfg.Tracer.Start(ctx, "GetPage",
@@ -365,7 +401,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
c.errors.Add(ctx, 1, otelAttrs...)
c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
span.End()
}()
@@ -395,7 +431,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
uri.AddPathParts(u, pathParts[:]...)
stage = "EncodeRequest"
r, err := ht.NewRequest(ctx, "GET", u, nil)
r, err := ht.NewRequest(ctx, "GET", u)
if err != nil {
return res, errors.Wrap(err, "create request")
}
@@ -423,24 +459,26 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get
// GET /pages
func (c *Client) GetPages(ctx context.Context) (Pages, error) {
res, err := c.sendGetPages(ctx)
_ = res
return res, err
}
func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
otelAttrs := []attribute.KeyValue{
otelogen.OperationID("getPages"),
semconv.HTTPMethodKey.String("GET"),
semconv.HTTPRouteKey.String("/pages"),
}
// Run stopwatch.
startTime := time.Now()
defer func() {
// Use floating point division here for higher precision (instead of Millisecond method).
elapsedDuration := time.Since(startTime)
c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
c.requests.Add(ctx, 1, otelAttrs...)
c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
// Start a span for this request.
ctx, span := c.cfg.Tracer.Start(ctx, "GetPages",
@@ -453,7 +491,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
c.errors.Add(ctx, 1, otelAttrs...)
c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
span.End()
}()
@@ -465,7 +503,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) {
uri.AddPathParts(u, pathParts[:]...)
stage = "EncodeRequest"
r, err := ht.NewRequest(ctx, "GET", u, nil)
r, err := ht.NewRequest(ctx, "GET", u)
if err != nil {
return res, errors.Wrap(err, "create request")
}

123
api/openapi/oas_handlers_gen.go
View File

@@ -10,7 +10,8 @@ import (
"github.com/go-faster/errors"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/metric"
semconv "go.opentelemetry.io/otel/semconv/v1.19.0"
"go.opentelemetry.io/otel/trace"
ht "github.com/ogen-go/ogen/http"
@@ -42,17 +43,18 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
startTime := time.Now()
defer func() {
elapsedDuration := time.Since(startTime)
s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
// Use floating point division here for higher precision (instead of Millisecond method).
s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
s.requests.Add(ctx, 1, otelAttrs...)
s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
var (
recordError = func(stage string, err error) {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
s.errors.Add(ctx, 1, otelAttrs...)
s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
err error
opErrContext = ogenerrors.OperationContext{
@@ -89,10 +91,11 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
var response AddPageRes
if m := s.cfg.Middleware; m != nil {
mreq := middleware.Request{
Context: ctx,
OperationName: "AddPage",
OperationID: "addPage",
Body: request,
Context: ctx,
OperationName: "AddPage",
OperationSummary: "Add new page",
OperationID: "addPage",
Body: request,
Params: middleware.Parameters{
{
Name: "url",
@@ -132,22 +135,27 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R
response, err = s.h.AddPage(ctx, request, params)
}
if err != nil {
recordError("Internal", err)
if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
encodeErrorResponse(errRes, w, span)
if err := encodeErrorResponse(errRes, w, span); err != nil {
recordError("Internal", err)
}
return
}
if errors.Is(err, ht.ErrNotImplemented) {
s.cfg.ErrorHandler(ctx, w, r, err)
return
}
encodeErrorResponse(s.h.NewError(ctx, err), w, span)
if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
recordError("Internal", err)
}
return
}
if err := encodeAddPageResponse(response, w, span); err != nil {
recordError("EncodeResponse", err)
s.cfg.ErrorHandler(ctx, w, r, err)
if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
s.cfg.ErrorHandler(ctx, w, r, err)
}
return
}
}
@@ -175,17 +183,18 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
startTime := time.Now()
defer func() {
elapsedDuration := time.Since(startTime)
s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
// Use floating point division here for higher precision (instead of Millisecond method).
s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
s.requests.Add(ctx, 1, otelAttrs...)
s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
var (
recordError = func(stage string, err error) {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
s.errors.Add(ctx, 1, otelAttrs...)
s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
err error
opErrContext = ogenerrors.OperationContext{
@@ -207,10 +216,11 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
var response GetFileRes
if m := s.cfg.Middleware; m != nil {
mreq := middleware.Request{
Context: ctx,
OperationName: "GetFile",
OperationID: "getFile",
Body: nil,
Context: ctx,
OperationName: "GetFile",
OperationSummary: "",
OperationID: "getFile",
Body: nil,
Params: middleware.Parameters{
{
Name: "id",
@@ -246,22 +256,27 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R
response, err = s.h.GetFile(ctx, params)
}
if err != nil {
recordError("Internal", err)
if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
encodeErrorResponse(errRes, w, span)
if err := encodeErrorResponse(errRes, w, span); err != nil {
recordError("Internal", err)
}
return
}
if errors.Is(err, ht.ErrNotImplemented) {
s.cfg.ErrorHandler(ctx, w, r, err)
return
}
encodeErrorResponse(s.h.NewError(ctx, err), w, span)
if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
recordError("Internal", err)
}
return
}
if err := encodeGetFileResponse(response, w, span); err != nil {
recordError("EncodeResponse", err)
s.cfg.ErrorHandler(ctx, w, r, err)
if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
s.cfg.ErrorHandler(ctx, w, r, err)
}
return
}
}
@@ -289,17 +304,18 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
startTime := time.Now()
defer func() {
elapsedDuration := time.Since(startTime)
s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
// Use floating point division here for higher precision (instead of Millisecond method).
s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
s.requests.Add(ctx, 1, otelAttrs...)
s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
var (
recordError = func(stage string, err error) {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
s.errors.Add(ctx, 1, otelAttrs...)
s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
err error
opErrContext = ogenerrors.OperationContext{
@@ -321,10 +337,11 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
var response GetPageRes
if m := s.cfg.Middleware; m != nil {
mreq := middleware.Request{
Context: ctx,
OperationName: "GetPage",
OperationID: "getPage",
Body: nil,
Context: ctx,
OperationName: "GetPage",
OperationSummary: "",
OperationID: "getPage",
Body: nil,
Params: middleware.Parameters{
{
Name: "id",
@@ -356,22 +373,27 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R
response, err = s.h.GetPage(ctx, params)
}
if err != nil {
recordError("Internal", err)
if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
encodeErrorResponse(errRes, w, span)
if err := encodeErrorResponse(errRes, w, span); err != nil {
recordError("Internal", err)
}
return
}
if errors.Is(err, ht.ErrNotImplemented) {
s.cfg.ErrorHandler(ctx, w, r, err)
return
}
encodeErrorResponse(s.h.NewError(ctx, err), w, span)
if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
recordError("Internal", err)
}
return
}
if err := encodeGetPageResponse(response, w, span); err != nil {
recordError("EncodeResponse", err)
s.cfg.ErrorHandler(ctx, w, r, err)
if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
s.cfg.ErrorHandler(ctx, w, r, err)
}
return
}
}
@@ -399,17 +421,18 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
startTime := time.Now()
defer func() {
elapsedDuration := time.Since(startTime)
s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...)
// Use floating point division here for higher precision (instead of Millisecond method).
s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...))
}()
// Increment request counter.
s.requests.Add(ctx, 1, otelAttrs...)
s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
var (
recordError = func(stage string, err error) {
span.RecordError(err)
span.SetStatus(codes.Error, stage)
s.errors.Add(ctx, 1, otelAttrs...)
s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...))
}
err error
)
@@ -417,12 +440,13 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
var response Pages
if m := s.cfg.Middleware; m != nil {
mreq := middleware.Request{
Context: ctx,
OperationName: "GetPages",
OperationID: "getPages",
Body: nil,
Params: middleware.Parameters{},
Raw: r,
Context: ctx,
OperationName: "GetPages",
OperationSummary: "Get all pages",
OperationID: "getPages",
Body: nil,
Params: middleware.Parameters{},
Raw: r,
}
type (
@@ -447,22 +471,27 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http.
response, err = s.h.GetPages(ctx)
}
if err != nil {
recordError("Internal", err)
if errRes, ok := errors.Into[*ErrorStatusCode](err); ok {
encodeErrorResponse(errRes, w, span)
if err := encodeErrorResponse(errRes, w, span); err != nil {
recordError("Internal", err)
}
return
}
if errors.Is(err, ht.ErrNotImplemented) {
s.cfg.ErrorHandler(ctx, w, r, err)
return
}
encodeErrorResponse(s.h.NewError(ctx, err), w, span)
if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil {
recordError("Internal", err)
}
return
}
if err := encodeGetPagesResponse(response, w, span); err != nil {
recordError("EncodeResponse", err)
s.cfg.ErrorHandler(ctx, w, r, err)
if !errors.Is(err, ht.ErrInternalServerErrorResponse) {
s.cfg.ErrorHandler(ctx, w, r, err)
}
return
}
}

27
api/openapi/oas_json_gen.go
View File

@@ -23,12 +23,10 @@ func (s *AddPageBadRequest) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *AddPageBadRequest) encodeFields(e *jx.Encoder) {
{
e.FieldStart("field")
e.Str(s.Field)
}
{
e.FieldStart("error")
e.Str(s.Error)
}
@@ -138,7 +136,6 @@ func (s *AddPageReq) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *AddPageReq) encodeFields(e *jx.Encoder) {
{
e.FieldStart("url")
e.Str(s.URL)
}
@@ -280,7 +277,6 @@ func (s *Error) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *Error) encodeFields(e *jx.Encoder) {
{
e.FieldStart("message")
e.Str(s.Message)
}
@@ -506,22 +502,18 @@ func (s *Page) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *Page) encodeFields(e *jx.Encoder) {
{
e.FieldStart("id")
json.EncodeUUID(e, s.ID)
}
{
e.FieldStart("url")
e.Str(s.URL)
}
{
e.FieldStart("created")
json.EncodeDateTime(e, s.Created)
}
{
e.FieldStart("formats")
e.ArrStart()
for _, elem := range s.Formats {
@@ -530,12 +522,10 @@ func (s *Page) encodeFields(e *jx.Encoder) {
e.ArrEnd()
}
{
e.FieldStart("status")
s.Status.Encode(e)
}
{
e.FieldStart("meta")
s.Meta.Encode(e)
}
@@ -699,12 +689,10 @@ func (s *PageMeta) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *PageMeta) encodeFields(e *jx.Encoder) {
{
e.FieldStart("title")
e.Str(s.Title)
}
{
e.FieldStart("description")
e.Str(s.Description)
}
@@ -831,22 +819,18 @@ func (s *PageWithResults) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *PageWithResults) encodeFields(e *jx.Encoder) {
{
e.FieldStart("id")
json.EncodeUUID(e, s.ID)
}
{
e.FieldStart("url")
e.Str(s.URL)
}
{
e.FieldStart("created")
json.EncodeDateTime(e, s.Created)
}
{
e.FieldStart("formats")
e.ArrStart()
for _, elem := range s.Formats {
@@ -855,17 +839,14 @@ func (s *PageWithResults) encodeFields(e *jx.Encoder) {
e.ArrEnd()
}
{
e.FieldStart("status")
s.Status.Encode(e)
}
{
e.FieldStart("meta")
s.Meta.Encode(e)
}
{
e.FieldStart("results")
e.ArrStart()
for _, elem := range s.Results {
@@ -1052,12 +1033,10 @@ func (s *PageWithResultsMeta) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *PageWithResultsMeta) encodeFields(e *jx.Encoder) {
{
e.FieldStart("title")
e.Str(s.Title)
}
{
e.FieldStart("description")
e.Str(s.Description)
}
@@ -1234,7 +1213,6 @@ func (s *Result) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *Result) encodeFields(e *jx.Encoder) {
{
e.FieldStart("format")
s.Format.Encode(e)
}
@@ -1245,7 +1223,6 @@ func (s *Result) encodeFields(e *jx.Encoder) {
}
}
{
e.FieldStart("files")
e.ArrStart()
for _, elem := range s.Files {
@@ -1374,22 +1351,18 @@ func (s *ResultFilesItem) Encode(e *jx.Encoder) {
// encodeFields encodes fields.
func (s *ResultFilesItem) encodeFields(e *jx.Encoder) {
{
e.FieldStart("id")
json.EncodeUUID(e, s.ID)
}
{
e.FieldStart("name")
e.Str(s.Name)
}
{
e.FieldStart("mimetype")
e.Str(s.Mimetype)
}
{
e.FieldStart("size")
e.Int64(s.Size)
}

4
api/openapi/oas_request_decoders_gen.go
View File

@@ -77,9 +77,9 @@ func (s *Server) decodeAddPageRequest(r *http.Request) (
return req, close, err
}
if err := func() error {
if request.Set {
if value, ok := request.Get(); ok {
if err := func() error {
if err := request.Value.Validate(); err != nil {
if err := value.Validate(); err != nil {
return err
}
return nil

2
api/openapi/oas_request_encoders_gen.go
View File

@@ -20,7 +20,7 @@ func encodeAddPageRequest(
// Keep request with empty body if value is not set.
return nil
}
e := jx.GetEncoder()
e := new(jx.Encoder)
{
if req.Set {
req.Encode(e)

16
api/openapi/oas_response_decoders_gen.go
View File

@@ -15,7 +15,7 @@ import (
"github.com/ogen-go/ogen/validate"
)
func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
func decodeAddPageResponse(resp *http.Response) (res AddPageRes, _ error) {
switch resp.StatusCode {
case 201:
// Code 201.
@@ -128,12 +128,12 @@ func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) {
}
}()
if err != nil {
return res, errors.Wrap(err, "default")
return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
}
return res, errors.Wrap(defRes, "error")
}
func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
func decodeGetFileResponse(resp *http.Response) (res GetFileRes, _ error) {
switch resp.StatusCode {
case 200:
// Code 200.
@@ -216,12 +216,12 @@ func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) {
}
}()
if err != nil {
return res, errors.Wrap(err, "default")
return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
}
return res, errors.Wrap(defRes, "error")
}
func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
func decodeGetPageResponse(resp *http.Response) (res GetPageRes, _ error) {
switch resp.StatusCode {
case 200:
// Code 200.
@@ -302,12 +302,12 @@ func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) {
}
}()
if err != nil {
return res, errors.Wrap(err, "default")
return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
}
return res, errors.Wrap(defRes, "error")
}
func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
func decodeGetPagesResponse(resp *http.Response) (res Pages, _ error) {
switch resp.StatusCode {
case 200:
// Code 200.
@@ -385,7 +385,7 @@ func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) {
}
}()
if err != nil {
return res, errors.Wrap(err, "default")
return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode)
}
return res, errors.Wrap(defRes, "error")
}

23
api/openapi/oas_response_encoders_gen.go
View File

@@ -10,6 +10,8 @@ import (
"github.com/go-faster/jx"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
ht "github.com/ogen-go/ogen/http"
)
func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trace.Span) error {
@@ -19,11 +21,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
w.WriteHeader(201)
span.SetStatus(codes.Ok, http.StatusText(201))
e := jx.GetEncoder()
e := new(jx.Encoder)
response.Encode(e)
if _, err := e.WriteTo(w); err != nil {
return errors.Wrap(err, "write")
}
return nil
case *AddPageBadRequest:
@@ -31,11 +34,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac
w.WriteHeader(400)
span.SetStatus(codes.Error, http.StatusText(400))
e := jx.GetEncoder()
e := new(jx.Encoder)
response.Encode(e)
if _, err := e.WriteTo(w); err != nil {
return errors.Wrap(err, "write")
}
return nil
default:
@@ -54,6 +58,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
if _, err := io.Copy(writer, response); err != nil {
return errors.Wrap(err, "write")
}
return nil
case *GetFileOKTextHTML:
@@ -65,6 +70,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
if _, err := io.Copy(writer, response); err != nil {
return errors.Wrap(err, "write")
}
return nil
case *GetFileOKTextPlain:
@@ -76,6 +82,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac
if _, err := io.Copy(writer, response); err != nil {
return errors.Wrap(err, "write")
}
return nil
case *GetFileNotFound:
@@ -96,11 +103,12 @@ func encodeGetPageResponse(response GetPageRes, w http.ResponseWriter, span trac
w.WriteHeader(200)
span.SetStatus(codes.Ok, http.StatusText(200))
e := jx.GetEncoder()
e := new(jx.Encoder)
response.Encode(e)
if _, err := e.WriteTo(w); err != nil {
return errors.Wrap(err, "write")
}
return nil
case *GetPageNotFound:
@@ -119,11 +127,12 @@ func encodeGetPagesResponse(response Pages, w http.ResponseWriter, span trace.Sp
w.WriteHeader(200)
span.SetStatus(codes.Ok, http.StatusText(200))
e := jx.GetEncoder()
e := new(jx.Encoder)
response.Encode(e)
if _, err := e.WriteTo(w); err != nil {
return errors.Wrap(err, "write")
}
return nil
}
@@ -142,11 +151,15 @@ func encodeErrorResponse(response *ErrorStatusCode, w http.ResponseWriter, span
span.SetStatus(codes.Ok, st)
}
e := jx.GetEncoder()
e := new(jx.Encoder)
response.Response.Encode(e)
if _, err := e.WriteTo(w); err != nil {
return errors.Wrap(err, "write")
}
if code >= http.StatusInternalServerError {
return errors.Wrapf(ht.ErrInternalServerErrorResponse, "code: %d, message: %s", code, http.StatusText(code))
}
return nil
}

42
api/openapi/oas_router_gen.go
View File

@@ -10,6 +10,19 @@ import (
"github.com/ogen-go/ogen/uri"
)
func (s *Server) cutPrefix(path string) (string, bool) {
prefix := s.cfg.Prefix
if prefix == "" {
return path, true
}
if !strings.HasPrefix(path, prefix) {
// Prefix doesn't match.
return "", false
}
// Cut prefix from the path.
return strings.TrimPrefix(path, prefix), true
}
// ServeHTTP serves http request as defined by OpenAPI v3 specification,
// calling handler that matches the path or returning not found error.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -21,17 +34,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
elemIsEscaped = strings.ContainsRune(elem, '%')
}
}
if prefix := s.cfg.Prefix; len(prefix) > 0 {
if strings.HasPrefix(elem, prefix) {
// Cut prefix from the path.
elem = strings.TrimPrefix(elem, prefix)
} else {
// Prefix doesn't match.
s.notFound(w, r)
return
}
}
if len(elem) == 0 {
elem, ok := s.cutPrefix(elem)
if !ok || len(elem) == 0 {
s.notFound(w, r)
return
}
@@ -129,6 +134,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Route is route object.
type Route struct {
name string
summary string
operationID string
pathPattern string
count int
@@ -142,6 +148,11 @@ func (r Route) Name() string {
return r.name
}
// Summary returns OpenAPI summary.
func (r Route) Summary() string {
return r.summary
}
// OperationID returns OpenAPI operationId.
func (r Route) OperationID() string {
return r.operationID
@@ -183,6 +194,11 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
}()
}
elem, ok := s.cutPrefix(elem)
if !ok {
return r, false
}
// Static code generated router with unwrapped path search.
switch {
default:
@@ -201,6 +217,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
switch method {
case "GET":
r.name = "GetPages"
r.summary = "Get all pages"
r.operationID = "getPages"
r.pathPattern = "/pages"
r.args = args
@@ -208,6 +225,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
return r, true
case "POST":
r.name = "AddPage"
r.summary = "Add new page"
r.operationID = "addPage"
r.pathPattern = "/pages"
r.args = args
@@ -238,6 +256,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
switch method {
case "GET":
r.name = "GetPage"
r.summary = ""
r.operationID = "getPage"
r.pathPattern = "/pages/{id}"
r.args = args
@@ -265,6 +284,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) {
case "GET":
// Leaf: GetFile
r.name = "GetFile"
r.summary = ""
r.operationID = "getFile"
r.pathPattern = "/pages/{id}/file/{file_id}"
r.args = args

30
api/openapi/oas_schemas_gen.go
View File

@@ -140,6 +140,16 @@ const (
FormatHeaders Format = "headers"
)
// AllValues returns all Format values.
func (Format) AllValues() []Format {
return []Format{
FormatAll,
FormatPdf,
FormatSingleFile,
FormatHeaders,
}
}
// MarshalText implements encoding.TextMarshaler.
func (s Format) MarshalText() ([]byte, error) {
switch s {
@@ -189,6 +199,9 @@ type GetFileOKApplicationPdf struct {
//
// Kept to satisfy the io.Reader interface.
func (s GetFileOKApplicationPdf) Read(p []byte) (n int, err error) {
if s.Data == nil {
return 0, io.EOF
}
return s.Data.Read(p)
}
@@ -202,6 +215,9 @@ type GetFileOKTextHTML struct {
//
// Kept to satisfy the io.Reader interface.
func (s GetFileOKTextHTML) Read(p []byte) (n int, err error) {
if s.Data == nil {
return 0, io.EOF
}
return s.Data.Read(p)
}
@@ -215,6 +231,9 @@ type GetFileOKTextPlain struct {
//
// Kept to satisfy the io.Reader interface.
func (s GetFileOKTextPlain) Read(p []byte) (n int, err error) {
if s.Data == nil {
return 0, io.EOF
}
return s.Data.Read(p)
}
@@ -642,6 +661,17 @@ const (
StatusWithErrors Status = "with_errors"
)
// AllValues returns all Status values.
func (Status) AllValues() []Status {
return []Status{
StatusNew,
StatusProcessing,
StatusDone,
StatusFailed,
StatusWithErrors,
}
}
// MarshalText implements encoding.TextMarshaler.
func (s Status) MarshalText() ([]byte, error) {
switch s {

10
api/openapi/oas_validators_gen.go
View File

@@ -42,6 +42,7 @@ func (s *AddPageReq) Validate() error {
}
return nil
}
func (s Format) Validate() error {
switch s {
case "all":
@@ -103,6 +104,7 @@ func (s *Page) Validate() error {
}
return nil
}
func (s *PageWithResults) Validate() error {
var failures []validate.FieldError
if err := func() error {
@@ -177,12 +179,14 @@ func (s *PageWithResults) Validate() error {
}
return nil
}
func (s Pages) Validate() error {
if s == nil {
alias := ([]Page)(s)
if alias == nil {
return errors.New("nil is invalid value")
}
var failures []validate.FieldError
for i, elem := range s {
for i, elem := range alias {
if err := func() error {
if err := elem.Validate(); err != nil {
return err
@@ -200,6 +204,7 @@ func (s Pages) Validate() error {
}
return nil
}
func (s *Result) Validate() error {
var failures []validate.FieldError
if err := func() error {
@@ -229,6 +234,7 @@ func (s *Result) Validate() error {
}
return nil
}
func (s Status) Validate() error {
switch s {
case "new":

5
application/application.go
View File

@@ -10,6 +10,7 @@ import (
"sync"
"time"
"github.com/derfenix/webarchive/adapters/repository"
"github.com/dgraph-io/badger/v4"
"github.com/ogen-go/ogen/middleware"
"go.uber.org/multierr"
@@ -30,7 +31,7 @@ func NewApplication(cfg config.Config) (Application, error) {
return Application{}, fmt.Errorf("new logger: %w", err)
}
db, err := badgerRepo.NewBadger(cfg.DB.Path, log.Named("db"))
db, err := repository.NewBadger(cfg.DB.Path, log.Named("db"))
if err != nil {
return Application{}, fmt.Errorf("new badger: %w", err)
}
@@ -173,7 +174,7 @@ func (a *Application) Stop() error {
errs = multierr.Append(errs, fmt.Errorf("sync db: %w", err))
}
if err := badgerRepo.Backup(a.db, badgerRepo.BackupStop); err != nil {
if err := repository.Backup(a.db, repository.BackupStop); err != nil {
errs = multierr.Append(errs, fmt.Errorf("backup on stop: %w", err))
}

32
entity/page.go
View File

@@ -45,29 +45,35 @@ type Meta struct {
Error string
}
func NewPage(url string, description string, formats ...Format) *Page {
return &Page{
ID: uuid.New(),
URL: url,
Description: description,
Formats: formats,
Created: time.Now(),
Version: 1,
}
}
type Page struct {
type PageBase struct {
ID uuid.UUID
URL string
Description string
Created time.Time
Formats []Format
Results ResultsRO
Version uint16
Status Status
Meta Meta
}
func NewPage(url string, description string, formats ...Format) *Page {
return &Page{
PageBase: PageBase{
ID: uuid.New(),
URL: url,
Description: description,
Formats: formats,
Created: time.Now(),
Version: 1,
},
}
}
type Page struct {
PageBase
Results ResultsRO
}
func (p *Page) SetProcessing() {
p.Status = StatusProcessing
}

49
go.mod
View File

@@ -7,32 +7,32 @@ require (
github.com/dgraph-io/badger/v4 v4.0.1
github.com/gabriel-vasile/mimetype v1.4.2
github.com/go-faster/errors v0.6.1
github.com/go-faster/jx v1.0.0
github.com/google/uuid v1.3.0
github.com/ogen-go/ogen v0.60.1
github.com/go-faster/jx v1.1.0
github.com/google/uuid v1.4.0
github.com/minio/minio-go/v7 v7.0.52
github.com/ogen-go/ogen v0.77.0
github.com/sethvargo/go-envconfig v0.9.0
github.com/stretchr/testify v1.8.2
github.com/stretchr/testify v1.8.4
github.com/vmihailenco/msgpack/v5 v5.3.5
go.opentelemetry.io/otel v1.14.0
go.opentelemetry.io/otel/metric v0.37.0
go.opentelemetry.io/otel/trace v1.14.0
go.uber.org/multierr v1.10.0
go.uber.org/zap v1.24.0
golang.org/x/net v0.8.0
go.opentelemetry.io/otel v1.19.0
go.opentelemetry.io/otel/metric v1.19.0
go.opentelemetry.io/otel/trace v1.19.0
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.26.0
golang.org/x/net v0.17.0
)
require (
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dlclark/regexp2 v1.8.1 // indirect
github.com/dlclark/regexp2 v1.10.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-faster/yamlx v0.4.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-faster/yaml v0.4.6 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.1.1 // indirect
@@ -40,20 +40,29 @@ require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/flatbuffers v23.3.3+incompatible // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.16.3 // indirect
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/xid v1.4.0 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

110
go.sum
View File

@@ -4,8 +4,6 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0 h1:DNrExYwvyyI404SxdUCCANAj9TwnGjRfa3cYFMNY1AU=
github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0/go.mod h1:SQq4xfIdvf6WYKSDxAJc+xOJdolt+/bc1jnQKMtPMvQ=
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@@ -23,8 +21,8 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0=
github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
@@ -40,13 +38,13 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI=
github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY=
github.com/go-faster/jx v1.0.0 h1:HE+ms2e6ZGkZ6u13t8u+onBinrPvIPI+0hWXGELm74g=
github.com/go-faster/jx v1.0.0/go.mod h1:zm8SlkwK+H0TYNKYtVJ/7cWFS7soJBQWhcPctKyYL/4=
github.com/go-faster/yamlx v0.4.1 h1:00RQkZopoLDF1SgBDJVHuN6epTOK7T0TkN427vbvEBk=
github.com/go-faster/yamlx v0.4.1/go.mod h1:QXr/i3Z00jRhskgyWkoGsEdseebd/ZbZEpGS6DJv8oo=
github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg=
github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg=
github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I=
github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -82,32 +80,54 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/ogen-go/ogen v0.60.1 h1:yOt0i6NcH7jM3rBi9nnv5VsGUQRw4ACUMsiJojnqrAM=
github.com/ogen-go/ogen v0.60.1/go.mod h1:tcwLpHe4vyk9xtbTMe3yu3Qtcbz8VjrpBz9LzsdwWvQ=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps=
github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/ogen-go/ogen v0.77.0 h1:yREPDg3cDuXkDyp7FPXdPEUz+azPZFUGKmYer8fJpmM=
github.com/ogen-go/ogen v0.77.0/go.mod h1:/bl+MubIppovr7F1fKAaDxzFF+oF2EiMtyVylyqDtQ8=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE=
github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -116,11 +136,12 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
@@ -129,25 +150,25 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM=
go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU=
go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs=
go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s=
go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg=
golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -162,28 +183,31 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -221,6 +245,8 @@ google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cn
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

23
ports/rest/converter.go
View File

@@ -66,6 +66,29 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults {
}
}
func BasePageToRest(page *entity.PageBase) openapi.Page {
return openapi.Page{
ID: page.ID,
URL: page.URL,
Created: page.Created,
Meta: openapi.PageMeta{
Title: html.EscapeString(page.Meta.Title),
Description: html.EscapeString(page.Meta.Description),
Error: openapi.NewOptString(page.Meta.Error),
},
Formats: func() []openapi.Format {
res := make([]openapi.Format, len(page.Formats))
for i, format := range page.Formats {
res[i] = FormatToRest(format)
}
return res
}(),
Status: StatusToRest(page.Status),
}
}
func PageToRest(page *entity.Page) openapi.Page {
return openapi.Page{
ID: page.ID,

2
ports/rest/service.go
View File

@@ -85,7 +85,7 @@ func (s *Service) AddPage(ctx context.Context, req openapi.OptAddPageReq, params
return nil, fmt.Errorf("save page: %w", err)
}
res := PageToRest(page)
res := BasePageToRest(&page.PageBase)
s.ch <- page