fenix/webarchive
1
0
Fork 0
mirror of https://github.com/derfenix/webarchive.git synced 2026-03-11 12:41:54 +03:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring

Browse Source
This commit is contained in:
derFenix
2023-04-15 20:10:58 +03:00
parent e1fbfe02d9
commit 1f3e5ec720
1 changed files with 20 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
adapters/processors/processors.go
View File

@@ -6,6 +6,7 @@ import (
"net" "net"
"net/http" "net/http"
"net/http/cookiejar" "net/http/cookiejar"
"strings"
"time" "time"
"golang.org/x/net/html" "golang.org/x/net/html"
@@ -14,6 +15,8 @@ import (
"github.com/derfenix/webarchive/entity" "github.com/derfenix/webarchive/entity"
) )
const defaultEncoding = "utf-8"
type processor interface { type processor interface {
Process(ctx context.Context, url string) ([]entity.File, error) Process(ctx context.Context, url string) ([]entity.File, error)
} }
@@ -128,6 +131,7 @@ func (p *Processors) GetMeta(ctx context.Context, url string) (entity.Meta, erro
meta := entity.Meta{} meta := entity.Meta{}
getMetaData(htmlNode, &meta) getMetaData(htmlNode, &meta)
meta.Encoding = encodingFromHeader(response.Header)
return meta, nil return meta, nil
} }
@@ -156,3 +160,19 @@ func getMetaData(n *html.Node, meta *entity.Meta) {
getMetaData(c, meta) getMetaData(c, meta)
} }
} }
func encodingFromHeader(headers http.Header) string {
var foundEncoding bool
var encoding string
_, encoding, foundEncoding = strings.Cut(headers.Get("Content-Type"), "; ")
if foundEncoding {
_, encoding, foundEncoding = strings.Cut(encoding, "=")
}
if !foundEncoding {
encoding = defaultEncoding
}
return encoding
}