grokify/mogo

package htmlutil

import (
    "bytes"
    "errors"
    "io"
    "os"

    "github.com/grokify/mogo/type/stringsutil"
    "golang.org/x/net/html"
)

var (
    ErrTokenNotFound           = errors.New("token(s) not found")
    ErrTokenizerNotInitialized = errors.New("tokenizer not initialized")
)

func NewTokenizerBytes(b []byte) *html.Tokenizer {
    return html.NewTokenizer(bytes.NewReader(b))
}

func NewTokenizerFile(name string) (*html.Tokenizer, error) {
    b, err := os.ReadFile(name)
    if err != nil {
        return nil, err
    }
    return NewTokenizerBytes(b), nil
}

/*
// TokensBetweenAtom returns the tokens that represent the `innerHtml`
// between a start and end tag token.
func TokensBetweenAtom(z *html.Tokenizer, skipErrors, inclusive bool, htmlAtom atom.Atom) ([]html.Token, error) {
    return TokensBetween(z, skipErrors, inclusive,
        TokenFilters{{
            TokenType: html.StartTagToken,
            AtomSet:   NewAtomSet(htmlAtom)}},
        TokenFilters{{
            TokenType: html.EndTagToken,
            AtomSet:   NewAtomSet(htmlAtom)}})
}
*/

/*
func TokensBetween(z *html.Tokenizer, skipErrors, inclusive bool, begin, end TokenFilters) ([]html.Token, error) {
    tokens := []html.Token{}
    tmsBegin, err := NextTokenMatch(z, skipErrors, false, false, begin...)
    if err != nil {
        return tokens, err
    }
    if inclusive {
        tokens = append(tokens, tmsBegin...)
    }
    tokensChain, err := NextTokenMatch(z, skipErrors, true, inclusive, end...)
    if err != nil {
        return tokens, err
    }
    tokens = append(tokens, tokensChain...)
    return tokens, nil
}
*/

/*

func TokensBetweenNew(z *html.Tokenizer, skipErrors, inclusive bool, begin, end []html.Token) ([]html.Token, error) {
    begFilters := Tokens(begin)
    endFilter := Tokens(end)

    tokens := []html.Token{}
    tmsBegin, err := NextToken(z, skipErrors, begin...)
    if err != nil {
        return tokens, err
    }
    if inclusive {
        tokens = append(tokens, tmsBegin...)
    }
    tokensChain, err := NextTokenMatch(z, skipErrors, true, inclusive, end...)
    if err != nil {
        return tokens, err
    }
    tokens = append(tokens, tokensChain...)
    return tokens, nil
}
*/

func NextToken(z *html.Tokenizer, skipErrors bool, tokFilters ...html.Token) (html.Token, error) {
    opts := NextTokensOpts{
        SkipErrors:     skipErrors,
        IncludeChain:   false,
        InclusiveMatch: true,
        StartFilter:    []html.Token{},
        EndFilter:      tokFilters,
    }
    toks, err := NextTokens(z, opts)
    if err != nil {
        return html.Token{}, err
    } else if len(toks) == 0 {
        return html.Token{}, ErrTokenNotFound
    } else if len(toks) > 1 {
        panic("too many tokens (>1) found")
    }
    return toks[0], nil
}

type NextTokensOpts struct {
    SkipErrors               bool
    IncludeChain             bool
    InclusiveMatch           bool
    StartFilter              Tokens
    StartAttributeValueMatch *stringsutil.MatchInfo
    EndFilter                Tokens
}

func NextTokens(z *html.Tokenizer, opts NextTokensOpts) (Tokens, error) {
    // func NextTokens(z *html.Tokenizer, skipErrors, includeChain, includeMatch bool, start, end []html.Token) ([]html.Token, error) {
    matches := []html.Token{}
    if z == nil {
        return matches, ErrTokenizerNotInitialized
    }
    if opts.StartAttributeValueMatch == nil {
        opts.StartAttributeValueMatch = &stringsutil.MatchInfo{
            MatchType: stringsutil.MatchExact,
        }
    }
    foundStartToken := false
    if len(opts.StartFilter) == 0 {
        foundStartToken = true
    }

    for {
        ttThis := z.Next()
        switch ttThis {
        case html.ErrorToken:
            err := z.Err()
            if err == io.EOF {
                return matches, nil
            } else if !opts.SkipErrors {
                return matches, err
            }
        default:
            tok := z.Token()
            if foundStartToken {
                if opts.EndFilter.MatchLeft(tok, nil) {
                    if opts.InclusiveMatch {
                        matches = append(matches, tok)
                    }
                    return matches, nil
                } else if opts.IncludeChain {
                    matches = append(matches, tok)
                }
            } else {
                if opts.StartFilter.MatchLeft(tok, opts.StartAttributeValueMatch) {
                    if opts.InclusiveMatch {
                        matches = append(matches, tok)
                    }
                    foundStartToken = true
                }
            }
        }
    }
}

// NextTextToken uses `NextTokensOpts` specifically for `SkipErrors`, `StartFilter`, and `StartAttributeValueMatch`.
func NextTextToken(z *html.Tokenizer, opts NextTokensOpts) (html.Token, error) {
    if z == nil {
        return html.Token{}, ErrTokenizerNotInitialized
    }
    if opts.StartAttributeValueMatch == nil {
        opts.StartAttributeValueMatch = &stringsutil.MatchInfo{
            MatchType: stringsutil.MatchExact,
        }
    }
    foundStartToken := false
    for {
        tokType := z.Next()
        tok := z.Token()
        if tokType == html.ErrorToken {
            err := z.Err()
            if err == io.EOF {
                return tok, ErrTokenNotFound
            } else if !opts.SkipErrors {
                return tok, err
            }
        } else if tokType == html.TextToken &&
            len(opts.StartFilter) == 0 || foundStartToken {
            return tok, nil
        } else if opts.StartFilter.MatchLeft(tok, opts.StartAttributeValueMatch) {
            foundStartToken = true
        }
    }
}

/*
// NextTokenMatch returns a string of matches. `includeMatch` is only used when `includeChain` is included.
func NextTokenMatch(z *html.Tokenizer, skipErrors, includeChain, includeMatch bool, filters ...TokenFilter) ([]html.Token, error) {
    matches := []html.Token{}
    if len(filters) == 0 {
        return matches, errors.New("no filters provided")
    }
    filtersMore := TokenFilters(filters)
    for {
        tt := z.Next()
        token := z.Token()
        if token.Type == html.ErrorToken {
            break
        }
        filtersForType := filtersMore.ByTokenType(tt)
        if len(filtersForType) > 0 {
            for _, filter := range filtersForType {
                if filter.AtomSet.Len() == 0 {
                    if !includeChain || includeMatch {
                        matches = append(matches, token)
                    }
                    return matches, nil
                } else if filter.AtomSet.Exists(token.DataAtom) {
                    if !includeChain || includeMatch {
                        matches = append(matches, token)
                    }
                    return matches, nil
                }
            }
        }
        if includeChain {
            matches = append(matches, token)
        }
    }
    return matches, nil
}
*/

/*
func NextStartToken(z *html.Tokenizer, skipErrors bool, htmlAtoms ...atom.Atom) (html.Token, error) {
    if len(htmlAtoms) == 0 {
        return html.Token{}, errors.New("no atoms requested")
    }
    atoms := NewAtomSet(htmlAtoms...)
    for {
        ttThis := z.Next()
        switch ttThis {
        case html.ErrorToken:
            err := z.Err()
            if z.Err() == io.EOF {
                return html.Token{}, ErrTokenNotFound
            } else if !skipErrors {
                return html.Token{}, err
            }
        case html.StartTagToken:
            tok := z.Token()
            if atoms.Exists(tok.DataAtom) {
                return tok, nil
            }
        }
    }
}
*/

/*
func NextTextToken(z *html.Tokenizer, skipErrors bool, htmlAtoms ...atom.Atom) (html.Token, error) {
    atoms := NewAtomSet(htmlAtoms...)
    for {
        tokType := z.Next()
        tok := z.Token()
        if tokType == html.ErrorToken {
            err := z.Err()
            if err == io.EOF {
                return tok, ErrTokenNotFound
            } else if !skipErrors {
                return tok, err
            }
        } else if atoms.Len() == 0 && tokType == html.TextToken {
            return tok, nil
        } else if atoms.Len() > 0 &&
            tokType == html.StartTagToken &&
            atoms.Exists(tok.DataAtom) {
            return NextTextToken(z, skipErrors)
        }
    }
}
*/

/*
func NextTextToken(z *html.Tokenizer, skipErrors bool, start []html.Token) (html.Token, error) {
    _, err := NextToken(z, skipErrors, start...)
    if err != nil {
        return html.Token{}, err
    }
    return NextToken(z, skipErrors, html.Token{Type: html.TextToken})
}
*/