web ui: index and basic details page, api refactoring

entity/page.go

@@ -11,6 +11,7 @@ import (
 
 type Processor interface {
 	Process(ctx context.Context, format Format, url string) Result
+	GetMeta(ctx context.Context, url string) (Meta, error)
 }
 
 type Format uint8
@@ -37,6 +38,12 @@ const (
 	StatusWithErrors
 )
 
+type Meta struct {
+	Title       string
+	Description string
+	Error       string
+}
+
 func NewPage(url string, description string, formats ...Format) *Page {
 	return &Page{
 		ID:          uuid.New(),
@@ -57,6 +64,7 @@ type Page struct {
 	Results     Results
 	Version     uint16
 	Status      Status
+	Meta        Meta
 }
 
 func (p *Page) SetProcessing() {

entity/worker.go

@@ -9,6 +9,7 @@ import (
 
 type Pages interface {
 	Save(ctx context.Context, page *Page) error
+	ListUnprocessed(ctx context.Context) ([]*Page, error)
 }
 
 func NewWorker(ch chan *Page, pages Pages, processor Processor, log *zap.Logger) *Worker {
@@ -27,6 +28,20 @@ func (w *Worker) Start(ctx context.Context, wg *sync.WaitGroup) {
 
 	w.log.Info("starting")
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		unprocessed, err := w.pages.ListUnprocessed(ctx)
+		if err != nil {
+			w.log.Error("failed to get unprocessed pages", zap.Error(err))
+		} else {
+			for i := range unprocessed {
+				w.ch <- unprocessed[i]
+			}
+		}
+	}()
+
 	for {
 		select {
 		case <-ctx.Done():