diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 91cf23e..db94ea1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,6 +1,7 @@ +--- name: release -on: +"on": push: tags: - 'v*' diff --git a/adapters/repository/badger/db.go b/adapters/repository/badger.go similarity index 99% rename from adapters/repository/badger/db.go rename to adapters/repository/badger.go index c1eb5cf..880a439 100644 --- a/adapters/repository/badger/db.go +++ b/adapters/repository/badger.go @@ -1,4 +1,4 @@ -package badger +package repository import ( "errors" diff --git a/adapters/repository/badger/page.go b/adapters/repository/badger/page.go index df1afe6..a6dcd7f 100644 --- a/adapters/repository/badger/page.go +++ b/adapters/repository/badger/page.go @@ -8,6 +8,8 @@ import ( "github.com/dgraph-io/badger/v4" "github.com/google/uuid" + "github.com/derfenix/webarchive/adapters/repository" + "github.com/derfenix/webarchive/entity" ) @@ -24,7 +26,9 @@ type Page struct { } func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.File, error) { - page := entity.Page{ID: pageID} + page := entity.Page{} + page.ID = pageID + var file *entity.File err := p.db.View(func(txn *badger.Txn) error { @@ -66,7 +70,7 @@ func (p *Page) GetFile(_ context.Context, pageID, fileID uuid.UUID) (*entity.Fil func (p *Page) Save(_ context.Context, page *entity.Page) error { if p.db.IsClosed() { - return ErrDBClosed + return repository.ErrDBClosed } marshaled, err := marshal(page) @@ -88,16 +92,17 @@ func (p *Page) Save(_ context.Context, page *entity.Page) error { } func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) { - site := entity.Page{ID: id} + page := entity.Page{} + page.ID = id err := p.db.View(func(txn *badger.Txn) error { - data, err := txn.Get(p.key(&site)) + data, err := txn.Get(p.key(&page)) if err != nil { return fmt.Errorf("get data: %w", err) } err = data.Value(func(val []byte) error { - if err := unmarshal(val, &site); err != nil { + if err := unmarshal(val, &page); err != nil { return fmt.Errorf("unmarshal data: %w", err) } @@ -113,7 +118,7 @@ func (p *Page) Get(_ context.Context, id uuid.UUID) (*entity.Page, error) { return nil, fmt.Errorf("view: %w", err) } - return &site, nil + return &page, nil } func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) { @@ -143,16 +148,7 @@ func (p *Page) ListAll(ctx context.Context) ([]*entity.Page, error) { return fmt.Errorf("get item: %w", err) } - pages = append(pages, &entity.Page{ - ID: page.ID, - URL: page.URL, - Description: page.Description, - Created: page.Created, - Formats: page.Formats, - Version: page.Version, - Status: page.Status, - Meta: page.Meta, - }) + pages = append(pages, &page) } return nil @@ -196,12 +192,11 @@ func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) { return fmt.Errorf("get item: %w", err) } - if page.Status != entity.StatusProcessing { - continue + if page.Status == entity.StatusNew { + //goland:noinspection GoVetCopyLock + pages = append(pages, page) //nolint:govet // didn't touch the lock here } - //goland:noinspection GoVetCopyLock - pages = append(pages, page) //nolint:govet // didn't touch the lock here } return nil diff --git a/adapters/repository/badger/page_test.go b/adapters/repository/badger/page_test.go index f977e88..8750294 100644 --- a/adapters/repository/badger/page_test.go +++ b/adapters/repository/badger/page_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/derfenix/webarchive/adapters/repository" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" @@ -31,7 +32,7 @@ func TestSite(t *testing.T) { log := zaptest.NewLogger(t) - db, err := NewBadger(tempDir, log.Named("db")) + db, err := repository.NewBadger(tempDir, log.Named("db")) require.NoError(t, err) siteRepo, err := NewPage(db) diff --git a/adapters/repository/badgers3/marshal.go b/adapters/repository/badgers3/marshal.go new file mode 100644 index 0000000..c5abb33 --- /dev/null +++ b/adapters/repository/badgers3/marshal.go @@ -0,0 +1,13 @@ +package badgers3 + +import ( + "github.com/vmihailenco/msgpack/v5" +) + +func marshal(v interface{}) ([]byte, error) { + return msgpack.Marshal(v) +} + +func unmarshal(b []byte, v interface{}) error { + return msgpack.Unmarshal(b, v) +} diff --git a/adapters/repository/badgers3/page.go b/adapters/repository/badgers3/page.go new file mode 100644 index 0000000..9a074c9 --- /dev/null +++ b/adapters/repository/badgers3/page.go @@ -0,0 +1,119 @@ +package badgers3 + +import ( + "bytes" + "context" + "errors" + "fmt" + "time" + + "github.com/derfenix/webarchive/adapters/repository" + "github.com/derfenix/webarchive/entity" + "github.com/dgraph-io/badger/v4" + "github.com/google/uuid" + "github.com/minio/minio-go/v7" +) + +func NewPage(db *badger.DB, s3 *minio.Client, bucketName string) (*Page, error) { + return &Page{ + db: db, + s3: s3, + prefix: []byte("pages3:"), + bucketName: bucketName, + }, nil +} + +type Page struct { + db *badger.DB + s3 *minio.Client + prefix []byte + bucketName string +} + +func (p *Page) ListAll(ctx context.Context) ([]*entity.PageBase, error) { + // TODO implement me + panic("implement me") +} + +func (p *Page) Get(ctx context.Context, id uuid.UUID) (*entity.Page, error) { + // TODO implement me + panic("implement me") +} + +func (p *Page) GetFile(ctx context.Context, pageID, fileID uuid.UUID) (*entity.File, error) { + // TODO implement me + panic("implement me") +} + +func (p *Page) Save(ctx context.Context, page *entity.Page) error { + if p.db.IsClosed() { + return repository.ErrDBClosed + } + + marshaled, err := marshal(page.PageBase) + if err != nil { + return fmt.Errorf("marshal data: %w", err) + } + + if err := p.db.Update(func(txn *badger.Txn) error { + if err := txn.Set(p.key(page), marshaled); err != nil { + return fmt.Errorf("put data: %w", err) + } + + return nil + }); err != nil { + return fmt.Errorf("update db: %w", err) + } + + snowball := make(chan minio.SnowballObject, 1) + + go func() { + defer close(snowball) + + for _, result := range page.Results { + for _, file := range result.Files { + for { + if ctx.Err() != nil { + return + } + + if len(snowball) < cap(snowball) { + break + } + } + + snowball <- minio.SnowballObject{ + Key: file.ID.String(), + Size: int64(len(file.Data)), + ModTime: time.Now(), + Content: bytes.NewReader(file.Data), + } + } + } + }() + + if err = p.s3.PutObjectsSnowball(ctx, p.bucketName, minio.SnowballOptions{Compress: true}, snowball); err != nil { + if dErr := p.db.Update(func(txn *badger.Txn) error { + if err := txn.Delete(p.key(page)); err != nil { + return fmt.Errorf("put data: %w", err) + } + + return nil + }); dErr != nil { + err = errors.Join(err, dErr) + } + + return fmt.Errorf("store files to s3: %w", err) + } + + return nil +} + +func (p *Page) ListUnprocessed(ctx context.Context) ([]entity.Page, error) { + // TODO implement me + panic("implement me") +} + +func (p *Page) key(site *entity.Page) []byte { + return append(p.prefix, []byte(site.ID.String())...) +} diff --git a/api/gen.go b/api/gen.go index 11fb9b7..3441b7d 100644 --- a/api/gen.go +++ b/api/gen.go @@ -1,3 +1,3 @@ package api -//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.60.1 --target ./openapi -package openapi --clean openapi.yaml +//go:generate go run github.com/ogen-go/ogen/cmd/ogen@v0.77.0 --target ./openapi -package openapi --clean openapi.yaml diff --git a/api/openapi/oas_cfg_gen.go b/api/openapi/oas_cfg_gen.go index 3bbddd8..8882ea5 100644 --- a/api/openapi/oas_cfg_gen.go +++ b/api/openapi/oas_cfg_gen.go @@ -7,7 +7,6 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/metric/instrument" "go.opentelemetry.io/otel/trace" ht "github.com/ogen-go/ogen/http" @@ -40,7 +39,7 @@ func (cfg *otelConfig) initOTEL() { cfg.TracerProvider = otel.GetTracerProvider() } if cfg.MeterProvider == nil { - cfg.MeterProvider = metric.NewNoopMeterProvider() + cfg.MeterProvider = otel.GetMeterProvider() } cfg.Tracer = cfg.TracerProvider.Tracer(otelogen.Name, trace.WithInstrumentationVersion(otelogen.SemVersion()), @@ -99,9 +98,9 @@ func newServerConfig(opts ...ServerOption) serverConfig { type baseServer struct { cfg serverConfig - requests instrument.Int64Counter - errors instrument.Int64Counter - duration instrument.Int64Histogram + requests metric.Int64Counter + errors metric.Int64Counter + duration metric.Float64Histogram } func (s baseServer) notFound(w http.ResponseWriter, r *http.Request) { @@ -120,7 +119,7 @@ func (cfg serverConfig) baseServer() (s baseServer, err error) { if s.errors, err = s.cfg.Meter.Int64Counter(otelogen.ServerErrorsCount); err != nil { return s, err } - if s.duration, err = s.cfg.Meter.Int64Histogram(otelogen.ServerDuration); err != nil { + if s.duration, err = s.cfg.Meter.Float64Histogram(otelogen.ServerDuration); err != nil { return s, err } return s, nil @@ -162,9 +161,9 @@ func newClientConfig(opts ...ClientOption) clientConfig { type baseClient struct { cfg clientConfig - requests instrument.Int64Counter - errors instrument.Int64Counter - duration instrument.Int64Histogram + requests metric.Int64Counter + errors metric.Int64Counter + duration metric.Float64Histogram } func (cfg clientConfig) baseClient() (c baseClient, err error) { @@ -175,7 +174,7 @@ func (cfg clientConfig) baseClient() (c baseClient, err error) { if c.errors, err = c.cfg.Meter.Int64Counter(otelogen.ClientErrorsCount); err != nil { return c, err } - if c.duration, err = c.cfg.Meter.Int64Histogram(otelogen.ClientDuration); err != nil { + if c.duration, err = c.cfg.Meter.Float64Histogram(otelogen.ClientDuration); err != nil { return c, err } return c, nil @@ -200,7 +199,7 @@ func WithTracerProvider(provider trace.TracerProvider) Option { // WithMeterProvider specifies a meter provider to use for creating a meter. // -// If none is specified, the metric.NewNoopMeterProvider is used. +// If none is specified, the otel.GetMeterProvider() is used. func WithMeterProvider(provider metric.MeterProvider) Option { return otelOptionFunc(func(cfg *otelConfig) { if provider != nil { diff --git a/api/openapi/oas_client_gen.go b/api/openapi/oas_client_gen.go index d34010c..6a603f4 100644 --- a/api/openapi/oas_client_gen.go +++ b/api/openapi/oas_client_gen.go @@ -11,6 +11,8 @@ import ( "github.com/go-faster/errors" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.19.0" "go.opentelemetry.io/otel/trace" "github.com/ogen-go/ogen/conv" @@ -19,6 +21,34 @@ import ( "github.com/ogen-go/ogen/uri" ) +// Invoker invokes operations described by OpenAPI v3 specification. +type Invoker interface { + // AddPage invokes addPage operation. + // + // Add new page. + // + // POST /pages + AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error) + // GetFile invokes getFile operation. + // + // Get file content. + // + // GET /pages/{id}/file/{file_id} + GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error) + // GetPage invokes getPage operation. + // + // Get page details. + // + // GET /pages/{id} + GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error) + // GetPages invokes getPages operation. + // + // Get all pages. + // + // GET /pages + GetPages(ctx context.Context) (Pages, error) +} + // Client implements OAS client. type Client struct { serverURL *url.URL @@ -78,19 +108,20 @@ func (c *Client) requestURL(ctx context.Context) *url.URL { // POST /pages func (c *Client) AddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (AddPageRes, error) { res, err := c.sendAddPage(ctx, request, params) - _ = res return res, err } func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params AddPageParams) (res AddPageRes, err error) { otelAttrs := []attribute.KeyValue{ otelogen.OperationID("addPage"), + semconv.HTTPMethodKey.String("POST"), + semconv.HTTPRouteKey.String("/pages"), } // Validate request before sending. if err := func() error { - if request.Set { + if value, ok := request.Get(); ok { if err := func() error { - if err := request.Value.Validate(); err != nil { + if err := value.Validate(); err != nil { return err } return nil @@ -106,12 +137,13 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params // Run stopwatch. startTime := time.Now() defer func() { + // Use floating point division here for higher precision (instead of Millisecond method). elapsedDuration := time.Since(startTime) - c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - c.requests.Add(ctx, 1, otelAttrs...) + c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) // Start a span for this request. ctx, span := c.cfg.Tracer.Start(ctx, "AddPage", @@ -124,7 +156,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params if err != nil { span.RecordError(err) span.SetStatus(codes.Error, stage) - c.errors.Add(ctx, 1, otelAttrs...) + c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } span.End() }() @@ -197,7 +229,7 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params u.RawQuery = q.Values().Encode() stage = "EncodeRequest" - r, err := ht.NewRequest(ctx, "POST", u, nil) + r, err := ht.NewRequest(ctx, "POST", u) if err != nil { return res, errors.Wrap(err, "create request") } @@ -228,24 +260,26 @@ func (c *Client) sendAddPage(ctx context.Context, request OptAddPageReq, params // GET /pages/{id}/file/{file_id} func (c *Client) GetFile(ctx context.Context, params GetFileParams) (GetFileRes, error) { res, err := c.sendGetFile(ctx, params) - _ = res return res, err } func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res GetFileRes, err error) { otelAttrs := []attribute.KeyValue{ otelogen.OperationID("getFile"), + semconv.HTTPMethodKey.String("GET"), + semconv.HTTPRouteKey.String("/pages/{id}/file/{file_id}"), } // Run stopwatch. startTime := time.Now() defer func() { + // Use floating point division here for higher precision (instead of Millisecond method). elapsedDuration := time.Since(startTime) - c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - c.requests.Add(ctx, 1, otelAttrs...) + c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) // Start a span for this request. ctx, span := c.cfg.Tracer.Start(ctx, "GetFile", @@ -258,7 +292,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get if err != nil { span.RecordError(err) span.SetStatus(codes.Error, stage) - c.errors.Add(ctx, 1, otelAttrs...) + c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } span.End() }() @@ -307,7 +341,7 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get uri.AddPathParts(u, pathParts[:]...) stage = "EncodeRequest" - r, err := ht.NewRequest(ctx, "GET", u, nil) + r, err := ht.NewRequest(ctx, "GET", u) if err != nil { return res, errors.Wrap(err, "create request") } @@ -335,24 +369,26 @@ func (c *Client) sendGetFile(ctx context.Context, params GetFileParams) (res Get // GET /pages/{id} func (c *Client) GetPage(ctx context.Context, params GetPageParams) (GetPageRes, error) { res, err := c.sendGetPage(ctx, params) - _ = res return res, err } func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res GetPageRes, err error) { otelAttrs := []attribute.KeyValue{ otelogen.OperationID("getPage"), + semconv.HTTPMethodKey.String("GET"), + semconv.HTTPRouteKey.String("/pages/{id}"), } // Run stopwatch. startTime := time.Now() defer func() { + // Use floating point division here for higher precision (instead of Millisecond method). elapsedDuration := time.Since(startTime) - c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - c.requests.Add(ctx, 1, otelAttrs...) + c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) // Start a span for this request. ctx, span := c.cfg.Tracer.Start(ctx, "GetPage", @@ -365,7 +401,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get if err != nil { span.RecordError(err) span.SetStatus(codes.Error, stage) - c.errors.Add(ctx, 1, otelAttrs...) + c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } span.End() }() @@ -395,7 +431,7 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get uri.AddPathParts(u, pathParts[:]...) stage = "EncodeRequest" - r, err := ht.NewRequest(ctx, "GET", u, nil) + r, err := ht.NewRequest(ctx, "GET", u) if err != nil { return res, errors.Wrap(err, "create request") } @@ -423,24 +459,26 @@ func (c *Client) sendGetPage(ctx context.Context, params GetPageParams) (res Get // GET /pages func (c *Client) GetPages(ctx context.Context) (Pages, error) { res, err := c.sendGetPages(ctx) - _ = res return res, err } func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) { otelAttrs := []attribute.KeyValue{ otelogen.OperationID("getPages"), + semconv.HTTPMethodKey.String("GET"), + semconv.HTTPRouteKey.String("/pages"), } // Run stopwatch. startTime := time.Now() defer func() { + // Use floating point division here for higher precision (instead of Millisecond method). elapsedDuration := time.Since(startTime) - c.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + c.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - c.requests.Add(ctx, 1, otelAttrs...) + c.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) // Start a span for this request. ctx, span := c.cfg.Tracer.Start(ctx, "GetPages", @@ -453,7 +491,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) { if err != nil { span.RecordError(err) span.SetStatus(codes.Error, stage) - c.errors.Add(ctx, 1, otelAttrs...) + c.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } span.End() }() @@ -465,7 +503,7 @@ func (c *Client) sendGetPages(ctx context.Context) (res Pages, err error) { uri.AddPathParts(u, pathParts[:]...) stage = "EncodeRequest" - r, err := ht.NewRequest(ctx, "GET", u, nil) + r, err := ht.NewRequest(ctx, "GET", u) if err != nil { return res, errors.Wrap(err, "create request") } diff --git a/api/openapi/oas_handlers_gen.go b/api/openapi/oas_handlers_gen.go index a5fd482..65ff709 100644 --- a/api/openapi/oas_handlers_gen.go +++ b/api/openapi/oas_handlers_gen.go @@ -10,7 +10,8 @@ import ( "github.com/go-faster/errors" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.17.0" + "go.opentelemetry.io/otel/metric" + semconv "go.opentelemetry.io/otel/semconv/v1.19.0" "go.opentelemetry.io/otel/trace" ht "github.com/ogen-go/ogen/http" @@ -42,17 +43,18 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R startTime := time.Now() defer func() { elapsedDuration := time.Since(startTime) - s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + // Use floating point division here for higher precision (instead of Millisecond method). + s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - s.requests.Add(ctx, 1, otelAttrs...) + s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) var ( recordError = func(stage string, err error) { span.RecordError(err) span.SetStatus(codes.Error, stage) - s.errors.Add(ctx, 1, otelAttrs...) + s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } err error opErrContext = ogenerrors.OperationContext{ @@ -89,10 +91,11 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R var response AddPageRes if m := s.cfg.Middleware; m != nil { mreq := middleware.Request{ - Context: ctx, - OperationName: "AddPage", - OperationID: "addPage", - Body: request, + Context: ctx, + OperationName: "AddPage", + OperationSummary: "Add new page", + OperationID: "addPage", + Body: request, Params: middleware.Parameters{ { Name: "url", @@ -132,22 +135,27 @@ func (s *Server) handleAddPageRequest(args [0]string, argsEscaped bool, w http.R response, err = s.h.AddPage(ctx, request, params) } if err != nil { - recordError("Internal", err) if errRes, ok := errors.Into[*ErrorStatusCode](err); ok { - encodeErrorResponse(errRes, w, span) + if err := encodeErrorResponse(errRes, w, span); err != nil { + recordError("Internal", err) + } return } if errors.Is(err, ht.ErrNotImplemented) { s.cfg.ErrorHandler(ctx, w, r, err) return } - encodeErrorResponse(s.h.NewError(ctx, err), w, span) + if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil { + recordError("Internal", err) + } return } if err := encodeAddPageResponse(response, w, span); err != nil { recordError("EncodeResponse", err) - s.cfg.ErrorHandler(ctx, w, r, err) + if !errors.Is(err, ht.ErrInternalServerErrorResponse) { + s.cfg.ErrorHandler(ctx, w, r, err) + } return } } @@ -175,17 +183,18 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R startTime := time.Now() defer func() { elapsedDuration := time.Since(startTime) - s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + // Use floating point division here for higher precision (instead of Millisecond method). + s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - s.requests.Add(ctx, 1, otelAttrs...) + s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) var ( recordError = func(stage string, err error) { span.RecordError(err) span.SetStatus(codes.Error, stage) - s.errors.Add(ctx, 1, otelAttrs...) + s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } err error opErrContext = ogenerrors.OperationContext{ @@ -207,10 +216,11 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R var response GetFileRes if m := s.cfg.Middleware; m != nil { mreq := middleware.Request{ - Context: ctx, - OperationName: "GetFile", - OperationID: "getFile", - Body: nil, + Context: ctx, + OperationName: "GetFile", + OperationSummary: "", + OperationID: "getFile", + Body: nil, Params: middleware.Parameters{ { Name: "id", @@ -246,22 +256,27 @@ func (s *Server) handleGetFileRequest(args [2]string, argsEscaped bool, w http.R response, err = s.h.GetFile(ctx, params) } if err != nil { - recordError("Internal", err) if errRes, ok := errors.Into[*ErrorStatusCode](err); ok { - encodeErrorResponse(errRes, w, span) + if err := encodeErrorResponse(errRes, w, span); err != nil { + recordError("Internal", err) + } return } if errors.Is(err, ht.ErrNotImplemented) { s.cfg.ErrorHandler(ctx, w, r, err) return } - encodeErrorResponse(s.h.NewError(ctx, err), w, span) + if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil { + recordError("Internal", err) + } return } if err := encodeGetFileResponse(response, w, span); err != nil { recordError("EncodeResponse", err) - s.cfg.ErrorHandler(ctx, w, r, err) + if !errors.Is(err, ht.ErrInternalServerErrorResponse) { + s.cfg.ErrorHandler(ctx, w, r, err) + } return } } @@ -289,17 +304,18 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R startTime := time.Now() defer func() { elapsedDuration := time.Since(startTime) - s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + // Use floating point division here for higher precision (instead of Millisecond method). + s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - s.requests.Add(ctx, 1, otelAttrs...) + s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) var ( recordError = func(stage string, err error) { span.RecordError(err) span.SetStatus(codes.Error, stage) - s.errors.Add(ctx, 1, otelAttrs...) + s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } err error opErrContext = ogenerrors.OperationContext{ @@ -321,10 +337,11 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R var response GetPageRes if m := s.cfg.Middleware; m != nil { mreq := middleware.Request{ - Context: ctx, - OperationName: "GetPage", - OperationID: "getPage", - Body: nil, + Context: ctx, + OperationName: "GetPage", + OperationSummary: "", + OperationID: "getPage", + Body: nil, Params: middleware.Parameters{ { Name: "id", @@ -356,22 +373,27 @@ func (s *Server) handleGetPageRequest(args [1]string, argsEscaped bool, w http.R response, err = s.h.GetPage(ctx, params) } if err != nil { - recordError("Internal", err) if errRes, ok := errors.Into[*ErrorStatusCode](err); ok { - encodeErrorResponse(errRes, w, span) + if err := encodeErrorResponse(errRes, w, span); err != nil { + recordError("Internal", err) + } return } if errors.Is(err, ht.ErrNotImplemented) { s.cfg.ErrorHandler(ctx, w, r, err) return } - encodeErrorResponse(s.h.NewError(ctx, err), w, span) + if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil { + recordError("Internal", err) + } return } if err := encodeGetPageResponse(response, w, span); err != nil { recordError("EncodeResponse", err) - s.cfg.ErrorHandler(ctx, w, r, err) + if !errors.Is(err, ht.ErrInternalServerErrorResponse) { + s.cfg.ErrorHandler(ctx, w, r, err) + } return } } @@ -399,17 +421,18 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http. startTime := time.Now() defer func() { elapsedDuration := time.Since(startTime) - s.duration.Record(ctx, elapsedDuration.Microseconds(), otelAttrs...) + // Use floating point division here for higher precision (instead of Millisecond method). + s.duration.Record(ctx, float64(float64(elapsedDuration)/float64(time.Millisecond)), metric.WithAttributes(otelAttrs...)) }() // Increment request counter. - s.requests.Add(ctx, 1, otelAttrs...) + s.requests.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) var ( recordError = func(stage string, err error) { span.RecordError(err) span.SetStatus(codes.Error, stage) - s.errors.Add(ctx, 1, otelAttrs...) + s.errors.Add(ctx, 1, metric.WithAttributes(otelAttrs...)) } err error ) @@ -417,12 +440,13 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http. var response Pages if m := s.cfg.Middleware; m != nil { mreq := middleware.Request{ - Context: ctx, - OperationName: "GetPages", - OperationID: "getPages", - Body: nil, - Params: middleware.Parameters{}, - Raw: r, + Context: ctx, + OperationName: "GetPages", + OperationSummary: "Get all pages", + OperationID: "getPages", + Body: nil, + Params: middleware.Parameters{}, + Raw: r, } type ( @@ -447,22 +471,27 @@ func (s *Server) handleGetPagesRequest(args [0]string, argsEscaped bool, w http. response, err = s.h.GetPages(ctx) } if err != nil { - recordError("Internal", err) if errRes, ok := errors.Into[*ErrorStatusCode](err); ok { - encodeErrorResponse(errRes, w, span) + if err := encodeErrorResponse(errRes, w, span); err != nil { + recordError("Internal", err) + } return } if errors.Is(err, ht.ErrNotImplemented) { s.cfg.ErrorHandler(ctx, w, r, err) return } - encodeErrorResponse(s.h.NewError(ctx, err), w, span) + if err := encodeErrorResponse(s.h.NewError(ctx, err), w, span); err != nil { + recordError("Internal", err) + } return } if err := encodeGetPagesResponse(response, w, span); err != nil { recordError("EncodeResponse", err) - s.cfg.ErrorHandler(ctx, w, r, err) + if !errors.Is(err, ht.ErrInternalServerErrorResponse) { + s.cfg.ErrorHandler(ctx, w, r, err) + } return } } diff --git a/api/openapi/oas_json_gen.go b/api/openapi/oas_json_gen.go index 0568e88..5cc6a7e 100644 --- a/api/openapi/oas_json_gen.go +++ b/api/openapi/oas_json_gen.go @@ -23,12 +23,10 @@ func (s *AddPageBadRequest) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *AddPageBadRequest) encodeFields(e *jx.Encoder) { { - e.FieldStart("field") e.Str(s.Field) } { - e.FieldStart("error") e.Str(s.Error) } @@ -138,7 +136,6 @@ func (s *AddPageReq) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *AddPageReq) encodeFields(e *jx.Encoder) { { - e.FieldStart("url") e.Str(s.URL) } @@ -280,7 +277,6 @@ func (s *Error) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *Error) encodeFields(e *jx.Encoder) { { - e.FieldStart("message") e.Str(s.Message) } @@ -506,22 +502,18 @@ func (s *Page) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *Page) encodeFields(e *jx.Encoder) { { - e.FieldStart("id") json.EncodeUUID(e, s.ID) } { - e.FieldStart("url") e.Str(s.URL) } { - e.FieldStart("created") json.EncodeDateTime(e, s.Created) } { - e.FieldStart("formats") e.ArrStart() for _, elem := range s.Formats { @@ -530,12 +522,10 @@ func (s *Page) encodeFields(e *jx.Encoder) { e.ArrEnd() } { - e.FieldStart("status") s.Status.Encode(e) } { - e.FieldStart("meta") s.Meta.Encode(e) } @@ -699,12 +689,10 @@ func (s *PageMeta) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *PageMeta) encodeFields(e *jx.Encoder) { { - e.FieldStart("title") e.Str(s.Title) } { - e.FieldStart("description") e.Str(s.Description) } @@ -831,22 +819,18 @@ func (s *PageWithResults) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *PageWithResults) encodeFields(e *jx.Encoder) { { - e.FieldStart("id") json.EncodeUUID(e, s.ID) } { - e.FieldStart("url") e.Str(s.URL) } { - e.FieldStart("created") json.EncodeDateTime(e, s.Created) } { - e.FieldStart("formats") e.ArrStart() for _, elem := range s.Formats { @@ -855,17 +839,14 @@ func (s *PageWithResults) encodeFields(e *jx.Encoder) { e.ArrEnd() } { - e.FieldStart("status") s.Status.Encode(e) } { - e.FieldStart("meta") s.Meta.Encode(e) } { - e.FieldStart("results") e.ArrStart() for _, elem := range s.Results { @@ -1052,12 +1033,10 @@ func (s *PageWithResultsMeta) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *PageWithResultsMeta) encodeFields(e *jx.Encoder) { { - e.FieldStart("title") e.Str(s.Title) } { - e.FieldStart("description") e.Str(s.Description) } @@ -1234,7 +1213,6 @@ func (s *Result) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *Result) encodeFields(e *jx.Encoder) { { - e.FieldStart("format") s.Format.Encode(e) } @@ -1245,7 +1223,6 @@ func (s *Result) encodeFields(e *jx.Encoder) { } } { - e.FieldStart("files") e.ArrStart() for _, elem := range s.Files { @@ -1374,22 +1351,18 @@ func (s *ResultFilesItem) Encode(e *jx.Encoder) { // encodeFields encodes fields. func (s *ResultFilesItem) encodeFields(e *jx.Encoder) { { - e.FieldStart("id") json.EncodeUUID(e, s.ID) } { - e.FieldStart("name") e.Str(s.Name) } { - e.FieldStart("mimetype") e.Str(s.Mimetype) } { - e.FieldStart("size") e.Int64(s.Size) } diff --git a/api/openapi/oas_request_decoders_gen.go b/api/openapi/oas_request_decoders_gen.go index bdd344b..5eeee1c 100644 --- a/api/openapi/oas_request_decoders_gen.go +++ b/api/openapi/oas_request_decoders_gen.go @@ -77,9 +77,9 @@ func (s *Server) decodeAddPageRequest(r *http.Request) ( return req, close, err } if err := func() error { - if request.Set { + if value, ok := request.Get(); ok { if err := func() error { - if err := request.Value.Validate(); err != nil { + if err := value.Validate(); err != nil { return err } return nil diff --git a/api/openapi/oas_request_encoders_gen.go b/api/openapi/oas_request_encoders_gen.go index f3f6f7c..f8f6603 100644 --- a/api/openapi/oas_request_encoders_gen.go +++ b/api/openapi/oas_request_encoders_gen.go @@ -20,7 +20,7 @@ func encodeAddPageRequest( // Keep request with empty body if value is not set. return nil } - e := jx.GetEncoder() + e := new(jx.Encoder) { if req.Set { req.Encode(e) diff --git a/api/openapi/oas_response_decoders_gen.go b/api/openapi/oas_response_decoders_gen.go index 77171cd..4ebf2f5 100644 --- a/api/openapi/oas_response_decoders_gen.go +++ b/api/openapi/oas_response_decoders_gen.go @@ -15,7 +15,7 @@ import ( "github.com/ogen-go/ogen/validate" ) -func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) { +func decodeAddPageResponse(resp *http.Response) (res AddPageRes, _ error) { switch resp.StatusCode { case 201: // Code 201. @@ -128,12 +128,12 @@ func decodeAddPageResponse(resp *http.Response) (res AddPageRes, err error) { } }() if err != nil { - return res, errors.Wrap(err, "default") + return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode) } return res, errors.Wrap(defRes, "error") } -func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) { +func decodeGetFileResponse(resp *http.Response) (res GetFileRes, _ error) { switch resp.StatusCode { case 200: // Code 200. @@ -216,12 +216,12 @@ func decodeGetFileResponse(resp *http.Response) (res GetFileRes, err error) { } }() if err != nil { - return res, errors.Wrap(err, "default") + return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode) } return res, errors.Wrap(defRes, "error") } -func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) { +func decodeGetPageResponse(resp *http.Response) (res GetPageRes, _ error) { switch resp.StatusCode { case 200: // Code 200. @@ -302,12 +302,12 @@ func decodeGetPageResponse(resp *http.Response) (res GetPageRes, err error) { } }() if err != nil { - return res, errors.Wrap(err, "default") + return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode) } return res, errors.Wrap(defRes, "error") } -func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) { +func decodeGetPagesResponse(resp *http.Response) (res Pages, _ error) { switch resp.StatusCode { case 200: // Code 200. @@ -385,7 +385,7 @@ func decodeGetPagesResponse(resp *http.Response) (res Pages, err error) { } }() if err != nil { - return res, errors.Wrap(err, "default") + return res, errors.Wrapf(err, "default (code %d)", resp.StatusCode) } return res, errors.Wrap(defRes, "error") } diff --git a/api/openapi/oas_response_encoders_gen.go b/api/openapi/oas_response_encoders_gen.go index 3285b30..3b215b0 100644 --- a/api/openapi/oas_response_encoders_gen.go +++ b/api/openapi/oas_response_encoders_gen.go @@ -10,6 +10,8 @@ import ( "github.com/go-faster/jx" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + + ht "github.com/ogen-go/ogen/http" ) func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trace.Span) error { @@ -19,11 +21,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac w.WriteHeader(201) span.SetStatus(codes.Ok, http.StatusText(201)) - e := jx.GetEncoder() + e := new(jx.Encoder) response.Encode(e) if _, err := e.WriteTo(w); err != nil { return errors.Wrap(err, "write") } + return nil case *AddPageBadRequest: @@ -31,11 +34,12 @@ func encodeAddPageResponse(response AddPageRes, w http.ResponseWriter, span trac w.WriteHeader(400) span.SetStatus(codes.Error, http.StatusText(400)) - e := jx.GetEncoder() + e := new(jx.Encoder) response.Encode(e) if _, err := e.WriteTo(w); err != nil { return errors.Wrap(err, "write") } + return nil default: @@ -54,6 +58,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac if _, err := io.Copy(writer, response); err != nil { return errors.Wrap(err, "write") } + return nil case *GetFileOKTextHTML: @@ -65,6 +70,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac if _, err := io.Copy(writer, response); err != nil { return errors.Wrap(err, "write") } + return nil case *GetFileOKTextPlain: @@ -76,6 +82,7 @@ func encodeGetFileResponse(response GetFileRes, w http.ResponseWriter, span trac if _, err := io.Copy(writer, response); err != nil { return errors.Wrap(err, "write") } + return nil case *GetFileNotFound: @@ -96,11 +103,12 @@ func encodeGetPageResponse(response GetPageRes, w http.ResponseWriter, span trac w.WriteHeader(200) span.SetStatus(codes.Ok, http.StatusText(200)) - e := jx.GetEncoder() + e := new(jx.Encoder) response.Encode(e) if _, err := e.WriteTo(w); err != nil { return errors.Wrap(err, "write") } + return nil case *GetPageNotFound: @@ -119,11 +127,12 @@ func encodeGetPagesResponse(response Pages, w http.ResponseWriter, span trace.Sp w.WriteHeader(200) span.SetStatus(codes.Ok, http.StatusText(200)) - e := jx.GetEncoder() + e := new(jx.Encoder) response.Encode(e) if _, err := e.WriteTo(w); err != nil { return errors.Wrap(err, "write") } + return nil } @@ -142,11 +151,15 @@ func encodeErrorResponse(response *ErrorStatusCode, w http.ResponseWriter, span span.SetStatus(codes.Ok, st) } - e := jx.GetEncoder() + e := new(jx.Encoder) response.Response.Encode(e) if _, err := e.WriteTo(w); err != nil { return errors.Wrap(err, "write") } + + if code >= http.StatusInternalServerError { + return errors.Wrapf(ht.ErrInternalServerErrorResponse, "code: %d, message: %s", code, http.StatusText(code)) + } return nil } diff --git a/api/openapi/oas_router_gen.go b/api/openapi/oas_router_gen.go index b741255..cff4a3d 100644 --- a/api/openapi/oas_router_gen.go +++ b/api/openapi/oas_router_gen.go @@ -10,6 +10,19 @@ import ( "github.com/ogen-go/ogen/uri" ) +func (s *Server) cutPrefix(path string) (string, bool) { + prefix := s.cfg.Prefix + if prefix == "" { + return path, true + } + if !strings.HasPrefix(path, prefix) { + // Prefix doesn't match. + return "", false + } + // Cut prefix from the path. + return strings.TrimPrefix(path, prefix), true +} + // ServeHTTP serves http request as defined by OpenAPI v3 specification, // calling handler that matches the path or returning not found error. func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -21,17 +34,9 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { elemIsEscaped = strings.ContainsRune(elem, '%') } } - if prefix := s.cfg.Prefix; len(prefix) > 0 { - if strings.HasPrefix(elem, prefix) { - // Cut prefix from the path. - elem = strings.TrimPrefix(elem, prefix) - } else { - // Prefix doesn't match. - s.notFound(w, r) - return - } - } - if len(elem) == 0 { + + elem, ok := s.cutPrefix(elem) + if !ok || len(elem) == 0 { s.notFound(w, r) return } @@ -129,6 +134,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Route is route object. type Route struct { name string + summary string operationID string pathPattern string count int @@ -142,6 +148,11 @@ func (r Route) Name() string { return r.name } +// Summary returns OpenAPI summary. +func (r Route) Summary() string { + return r.summary +} + // OperationID returns OpenAPI operationId. func (r Route) OperationID() string { return r.operationID @@ -183,6 +194,11 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) { }() } + elem, ok := s.cutPrefix(elem) + if !ok { + return r, false + } + // Static code generated router with unwrapped path search. switch { default: @@ -201,6 +217,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) { switch method { case "GET": r.name = "GetPages" + r.summary = "Get all pages" r.operationID = "getPages" r.pathPattern = "/pages" r.args = args @@ -208,6 +225,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) { return r, true case "POST": r.name = "AddPage" + r.summary = "Add new page" r.operationID = "addPage" r.pathPattern = "/pages" r.args = args @@ -238,6 +256,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) { switch method { case "GET": r.name = "GetPage" + r.summary = "" r.operationID = "getPage" r.pathPattern = "/pages/{id}" r.args = args @@ -265,6 +284,7 @@ func (s *Server) FindPath(method string, u *url.URL) (r Route, _ bool) { case "GET": // Leaf: GetFile r.name = "GetFile" + r.summary = "" r.operationID = "getFile" r.pathPattern = "/pages/{id}/file/{file_id}" r.args = args diff --git a/api/openapi/oas_schemas_gen.go b/api/openapi/oas_schemas_gen.go index 8cc81a8..ab045a3 100644 --- a/api/openapi/oas_schemas_gen.go +++ b/api/openapi/oas_schemas_gen.go @@ -140,6 +140,16 @@ const ( FormatHeaders Format = "headers" ) +// AllValues returns all Format values. +func (Format) AllValues() []Format { + return []Format{ + FormatAll, + FormatPdf, + FormatSingleFile, + FormatHeaders, + } +} + // MarshalText implements encoding.TextMarshaler. func (s Format) MarshalText() ([]byte, error) { switch s { @@ -189,6 +199,9 @@ type GetFileOKApplicationPdf struct { // // Kept to satisfy the io.Reader interface. func (s GetFileOKApplicationPdf) Read(p []byte) (n int, err error) { + if s.Data == nil { + return 0, io.EOF + } return s.Data.Read(p) } @@ -202,6 +215,9 @@ type GetFileOKTextHTML struct { // // Kept to satisfy the io.Reader interface. func (s GetFileOKTextHTML) Read(p []byte) (n int, err error) { + if s.Data == nil { + return 0, io.EOF + } return s.Data.Read(p) } @@ -215,6 +231,9 @@ type GetFileOKTextPlain struct { // // Kept to satisfy the io.Reader interface. func (s GetFileOKTextPlain) Read(p []byte) (n int, err error) { + if s.Data == nil { + return 0, io.EOF + } return s.Data.Read(p) } @@ -642,6 +661,17 @@ const ( StatusWithErrors Status = "with_errors" ) +// AllValues returns all Status values. +func (Status) AllValues() []Status { + return []Status{ + StatusNew, + StatusProcessing, + StatusDone, + StatusFailed, + StatusWithErrors, + } +} + // MarshalText implements encoding.TextMarshaler. func (s Status) MarshalText() ([]byte, error) { switch s { diff --git a/api/openapi/oas_validators_gen.go b/api/openapi/oas_validators_gen.go index a8c3490..fb00f1b 100644 --- a/api/openapi/oas_validators_gen.go +++ b/api/openapi/oas_validators_gen.go @@ -42,6 +42,7 @@ func (s *AddPageReq) Validate() error { } return nil } + func (s Format) Validate() error { switch s { case "all": @@ -103,6 +104,7 @@ func (s *Page) Validate() error { } return nil } + func (s *PageWithResults) Validate() error { var failures []validate.FieldError if err := func() error { @@ -177,12 +179,14 @@ func (s *PageWithResults) Validate() error { } return nil } + func (s Pages) Validate() error { - if s == nil { + alias := ([]Page)(s) + if alias == nil { return errors.New("nil is invalid value") } var failures []validate.FieldError - for i, elem := range s { + for i, elem := range alias { if err := func() error { if err := elem.Validate(); err != nil { return err @@ -200,6 +204,7 @@ func (s Pages) Validate() error { } return nil } + func (s *Result) Validate() error { var failures []validate.FieldError if err := func() error { @@ -229,6 +234,7 @@ func (s *Result) Validate() error { } return nil } + func (s Status) Validate() error { switch s { case "new": diff --git a/application/application.go b/application/application.go index 36336f7..70b4cce 100644 --- a/application/application.go +++ b/application/application.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/derfenix/webarchive/adapters/repository" "github.com/dgraph-io/badger/v4" "github.com/ogen-go/ogen/middleware" "go.uber.org/multierr" @@ -30,7 +31,7 @@ func NewApplication(cfg config.Config) (Application, error) { return Application{}, fmt.Errorf("new logger: %w", err) } - db, err := badgerRepo.NewBadger(cfg.DB.Path, log.Named("db")) + db, err := repository.NewBadger(cfg.DB.Path, log.Named("db")) if err != nil { return Application{}, fmt.Errorf("new badger: %w", err) } @@ -173,7 +174,7 @@ func (a *Application) Stop() error { errs = multierr.Append(errs, fmt.Errorf("sync db: %w", err)) } - if err := badgerRepo.Backup(a.db, badgerRepo.BackupStop); err != nil { + if err := repository.Backup(a.db, repository.BackupStop); err != nil { errs = multierr.Append(errs, fmt.Errorf("backup on stop: %w", err)) } diff --git a/entity/page.go b/entity/page.go index 8d98dc5..6ef14ff 100644 --- a/entity/page.go +++ b/entity/page.go @@ -45,29 +45,35 @@ type Meta struct { Error string } -func NewPage(url string, description string, formats ...Format) *Page { - return &Page{ - ID: uuid.New(), - URL: url, - Description: description, - Formats: formats, - Created: time.Now(), - Version: 1, - } -} - -type Page struct { +type PageBase struct { ID uuid.UUID URL string Description string Created time.Time Formats []Format - Results ResultsRO Version uint16 Status Status Meta Meta } +func NewPage(url string, description string, formats ...Format) *Page { + return &Page{ + PageBase: PageBase{ + ID: uuid.New(), + URL: url, + Description: description, + Formats: formats, + Created: time.Now(), + Version: 1, + }, + } +} + +type Page struct { + PageBase + Results ResultsRO +} + func (p *Page) SetProcessing() { p.Status = StatusProcessing } diff --git a/go.mod b/go.mod index 118a39a..2e2c828 100644 --- a/go.mod +++ b/go.mod @@ -7,32 +7,32 @@ require ( github.com/dgraph-io/badger/v4 v4.0.1 github.com/gabriel-vasile/mimetype v1.4.2 github.com/go-faster/errors v0.6.1 - github.com/go-faster/jx v1.0.0 - github.com/google/uuid v1.3.0 - github.com/ogen-go/ogen v0.60.1 + github.com/go-faster/jx v1.1.0 + github.com/google/uuid v1.4.0 + github.com/minio/minio-go/v7 v7.0.52 + github.com/ogen-go/ogen v0.77.0 github.com/sethvargo/go-envconfig v0.9.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/vmihailenco/msgpack/v5 v5.3.5 - go.opentelemetry.io/otel v1.14.0 - go.opentelemetry.io/otel/metric v0.37.0 - go.opentelemetry.io/otel/trace v1.14.0 - go.uber.org/multierr v1.10.0 - go.uber.org/zap v1.24.0 - golang.org/x/net v0.8.0 + go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/otel/metric v1.19.0 + go.opentelemetry.io/otel/trace v1.19.0 + go.uber.org/multierr v1.11.0 + go.uber.org/zap v1.26.0 + golang.org/x/net v0.17.0 ) require ( - github.com/benbjohnson/clock v1.3.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect - github.com/dlclark/regexp2 v1.8.1 // indirect + github.com/dlclark/regexp2 v1.10.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/color v1.15.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-faster/yamlx v0.4.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-faster/yaml v0.4.6 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.1.1 // indirect @@ -40,20 +40,29 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v23.3.3+incompatible // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.3 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rs/xid v1.4.0 // indirect github.com/segmentio/asm v1.2.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index dfbc794..9819d1b 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,6 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0 h1:DNrExYwvyyI404SxdUCCANAj9TwnGjRfa3cYFMNY1AU= github.com/SebastiaanKlippert/go-wkhtmltopdf v1.9.0/go.mod h1:SQq4xfIdvf6WYKSDxAJc+xOJdolt+/bc1jnQKMtPMvQ= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -23,8 +21,8 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= -github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -40,13 +38,13 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI= github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY= -github.com/go-faster/jx v1.0.0 h1:HE+ms2e6ZGkZ6u13t8u+onBinrPvIPI+0hWXGELm74g= -github.com/go-faster/jx v1.0.0/go.mod h1:zm8SlkwK+H0TYNKYtVJ/7cWFS7soJBQWhcPctKyYL/4= -github.com/go-faster/yamlx v0.4.1 h1:00RQkZopoLDF1SgBDJVHuN6epTOK7T0TkN427vbvEBk= -github.com/go-faster/yamlx v0.4.1/go.mod h1:QXr/i3Z00jRhskgyWkoGsEdseebd/ZbZEpGS6DJv8oo= +github.com/go-faster/jx v1.1.0 h1:ZsW3wD+snOdmTDy9eIVgQdjUpXRRV4rqW8NS3t+20bg= +github.com/go-faster/jx v1.1.0/go.mod h1:vKDNikrKoyUmpzaJ0OkIkRQClNHFX/nF3dnTJZb3skg= +github.com/go-faster/yaml v0.4.6 h1:lOK/EhI04gCpPgPhgt0bChS6bvw7G3WwI8xxVe0sw9I= +github.com/go-faster/yaml v0.4.6/go.mod h1:390dRIvV4zbnO7qC9FGo6YYutc+wyyUSHBgbXL52eXk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -82,32 +80,54 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/ogen-go/ogen v0.60.1 h1:yOt0i6NcH7jM3rBi9nnv5VsGUQRw4ACUMsiJojnqrAM= -github.com/ogen-go/ogen v0.60.1/go.mod h1:tcwLpHe4vyk9xtbTMe3yu3Qtcbz8VjrpBz9LzsdwWvQ= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.52 h1:8XhG36F6oKQUDDSuz6dY3rioMzovKjW40W6ANuN0Dps= +github.com/minio/minio-go/v7 v7.0.52/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/ogen-go/ogen v0.77.0 h1:yREPDg3cDuXkDyp7FPXdPEUz+azPZFUGKmYer8fJpmM= +github.com/ogen-go/ogen v0.77.0/go.mod h1:/bl+MubIppovr7F1fKAaDxzFF+oF2EiMtyVylyqDtQ8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE= github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -116,11 +136,12 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= @@ -129,25 +150,25 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= -go.opentelemetry.io/otel/metric v0.37.0 h1:pHDQuLQOZwYD+Km0eb657A25NaRzy0a+eLyKfDXedEs= -go.opentelemetry.io/otel/metric v0.37.0/go.mod h1:DmdaHfGt54iV6UKxsV9slj2bBRJcKC1B1uvDLIioc1s= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= +golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -162,28 +183,31 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -221,6 +245,8 @@ google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cn google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/ports/rest/converter.go b/ports/rest/converter.go index af3014a..5b0eb84 100644 --- a/ports/rest/converter.go +++ b/ports/rest/converter.go @@ -66,6 +66,29 @@ func PageToRestWithResults(page *entity.Page) openapi.PageWithResults { } } +func BasePageToRest(page *entity.PageBase) openapi.Page { + return openapi.Page{ + ID: page.ID, + URL: page.URL, + Created: page.Created, + Meta: openapi.PageMeta{ + Title: html.EscapeString(page.Meta.Title), + Description: html.EscapeString(page.Meta.Description), + Error: openapi.NewOptString(page.Meta.Error), + }, + Formats: func() []openapi.Format { + res := make([]openapi.Format, len(page.Formats)) + + for i, format := range page.Formats { + res[i] = FormatToRest(format) + } + + return res + }(), + Status: StatusToRest(page.Status), + } +} + func PageToRest(page *entity.Page) openapi.Page { return openapi.Page{ ID: page.ID, diff --git a/ports/rest/service.go b/ports/rest/service.go index 9192b7a..c8ed763 100644 --- a/ports/rest/service.go +++ b/ports/rest/service.go @@ -85,7 +85,7 @@ func (s *Service) AddPage(ctx context.Context, req openapi.OptAddPageReq, params return nil, fmt.Errorf("save page: %w", err) } - res := PageToRest(page) + res := BasePageToRest(&page.PageBase) s.ch <- page