mirror of
https://github.com/derfenix/webarchive.git
synced 2026-03-11 22:40:58 +03:00
127 lines
2.9 KiB
Go
127 lines
2.9 KiB
Go
package processors
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
|
|
"github.com/derfenix/webarchive/entity"
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
func NewSingleFile(client *http.Client) *SingleFile {
|
|
return &SingleFile{client: client}
|
|
}
|
|
|
|
type SingleFile struct {
|
|
client *http.Client
|
|
}
|
|
|
|
func (s *SingleFile) Process(ctx context.Context, url string) ([]entity.File, error) {
|
|
response, err := s.get(ctx, url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
htmlNode, err := html.Parse(response.Body)
|
|
if err != nil {
|
|
_ = response.Body.Close()
|
|
return nil, fmt.Errorf("parse response body: %w", err)
|
|
}
|
|
|
|
_ = response.Body.Close()
|
|
|
|
if err := s.process(ctx, htmlNode, url, response.Header); err != nil {
|
|
return nil, fmt.Errorf("process: %w", err)
|
|
}
|
|
|
|
buf := bytes.NewBuffer(nil)
|
|
if err := html.Render(buf, htmlNode); err != nil {
|
|
return nil, fmt.Errorf("render result html: %w", err)
|
|
}
|
|
|
|
htmlFile := entity.NewFile("page.html", buf.Bytes())
|
|
|
|
return []entity.File{htmlFile}, nil
|
|
}
|
|
|
|
func (s *SingleFile) get(ctx context.Context, url string) (*http.Response, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("new request: %w", err)
|
|
}
|
|
|
|
response, err := s.client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("do request: %w", err)
|
|
}
|
|
|
|
if response.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("want status 200, got %d", response.StatusCode)
|
|
}
|
|
|
|
if response.Body == nil {
|
|
return nil, fmt.Errorf("empty response body")
|
|
}
|
|
|
|
return response, nil
|
|
}
|
|
|
|
func (s *SingleFile) process(ctx context.Context, node *html.Node, pageURL string, headers http.Header) error {
|
|
parsedURL, err := url.Parse(pageURL)
|
|
if err != nil {
|
|
return fmt.Errorf("parse page url: %w", err)
|
|
}
|
|
|
|
baseURL := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
|
|
|
|
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
|
var err error
|
|
switch child.Data {
|
|
case "head":
|
|
err = s.processHead(ctx, child, baseURL)
|
|
|
|
case "body":
|
|
err = s.processBody(ctx, child, baseURL)
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *SingleFile) processHead(ctx context.Context, node *html.Node, baseURL string) error {
|
|
for child := node.FirstChild; child != nil; child = child.NextSibling {
|
|
switch child.Data {
|
|
case "link":
|
|
if err := s.processHref(ctx, child.Attr, baseURL); err != nil {
|
|
return fmt.Errorf("process link %s: %w", child.Attr, err)
|
|
}
|
|
|
|
case "script":
|
|
if err := s.processSrc(ctx, child.Attr, baseURL); err != nil {
|
|
return fmt.Errorf("process script %s: %w", child.Attr, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s *SingleFile) processBody(ctx context.Context, child *html.Node, url string) error {
|
|
return nil
|
|
}
|
|
|
|
func (s *SingleFile) processHref(ctx context.Context, attrs []html.Attribute, baseURL string) error {
|
|
return nil
|
|
}
|
|
|
|
func (s *SingleFile) processSrc(ctx context.Context, attrs []html.Attribute, baseURL string) error {
|
|
return nil
|
|
}
|