Fix reduce network calls count for the target url

entity/cache.go

@@ -0,0 +1,42 @@
+package entity
+
+import (
+	"bytes"
+	"io"
+	"sync"
+)
+
+func NewCache() *Cache {
+	return &Cache{data: make([]byte, 0, 1024*512)}
+}
+
+type Cache struct {
+	mu   sync.RWMutex
+	data []byte
+}
+
+func (c *Cache) Write(p []byte) (n int, err error) {
+	c.mu.Lock()
+	c.data = append(c.data, p...)
+	c.mu.Unlock()
+
+	return len(p), nil
+}
+
+func (c *Cache) Get() []byte {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	return c.data
+}
+
+func (c *Cache) Reader() io.Reader {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if len(c.data) == 0 {
+		return nil
+	}
+
+	return bytes.NewBuffer(c.data)
+}

entity/page.go

@@ -10,8 +10,8 @@ import (
 )
 
 type Processor interface {
-	Process(ctx context.Context, format Format, url string) Result
-	GetMeta(ctx context.Context, url string) (Meta, error)
+	Process(ctx context.Context, format Format, url string, cache *Cache) Result
+	GetMeta(ctx context.Context, url string, cache *Cache) (Meta, error)
 }
 
 type Format uint8
@@ -66,12 +66,14 @@ func NewPage(url string, description string, formats ...Format) *Page {
 			Created:     time.Now(),
 			Version:     1,
 		},
+		cache: NewCache(),
 	}
 }
 
 type Page struct {
 	PageBase
 	Results ResultsRO
+	cache   *Cache
 }
 
 func (p *Page) SetProcessing() {
@@ -82,7 +84,7 @@ func (p *Page) Process(ctx context.Context, processor Processor) {
 	innerWG := sync.WaitGroup{}
 	innerWG.Add(len(p.Formats))
 
-	meta, err := processor.GetMeta(ctx, p.URL)
+	meta, err := processor.GetMeta(ctx, p.URL, p.cache)
 	if err != nil {
 		p.Meta.Error = err.Error()
 	} else {
@@ -101,7 +103,7 @@ func (p *Page) Process(ctx context.Context, processor Processor) {
 				}
 			}()
 
-			result := processor.Process(ctx, format, p.URL)
+			result := processor.Process(ctx, format, p.URL, p.cache)
 			results.Add(result)
 		}(format)
 	}