Sign upLogin

nlpodyssey/gotokenizers

View on GitHub
Star

Showing 92 of 92 total issues

exported function NewDefault should have comment or be unexported
Open

func NewDefault() *WhiteSpacePreTokenizer {
Severity: Minor
Found in pretokenizers/whitespacepretokenizer/whitespacepretokenizer.go by golint

exported function Invert should have comment or be unexported
Open

func Invert(sp SplitPattern) *InvertedPattern {
Severity: Minor
Found in splitpattern/invertedpattern.go by golint

exported type WordMergeHeap should have comment or be unexported
Open

type WordMergeHeap []WordMerge
Severity: Minor
Found in models/bpemodel/wordmerge.go by golint

exported function New should have comment or be unexported
Open

func New(
Severity: Minor
Found in models/wordpiecemodel/wordpiecemodel.go by golint

exported method FuncSplitPattern.FindMatches should have comment or be unexported
Open

func (sp *FuncSplitPattern) FindMatches(s string) ([]Capture, error) {
Severity: Minor
Found in splitpattern/func.go by golint

exported type StringSplitPattern should have comment or be unexported
Open

type StringSplitPattern struct {
Severity: Minor
Found in splitpattern/string.go by golint

exported method StringSplitPattern.FindMatches should have comment or be unexported
Open

func (sp *StringSplitPattern) FindMatches(s string) ([]Capture, error) {
Severity: Minor
Found in splitpattern/string.go by golint

exported method Word.Len should have comment or be unexported
Open

func (w *Word) Len() int {
Severity: Minor
Found in models/bpemodel/word.go by golint

exported method WordCache.SetValues should have comment or be unexported
Open

func (c *WordCache) SetValues(keys []string, values []*Word) {
Severity: Minor
Found in models/bpemodel/wordcache.go by golint

exported method NormalizedRange.End should have comment or be unexported
Open

func (r NormalizedRange) End() int {
Severity: Minor
Found in normalizedstring/range.go by golint

exported type Token should have comment or be unexported
Open

type Token struct {
Severity: Minor
Found in models/models.go by golint

exported method NormalizedRange.Start should have comment or be unexported
Open

func (r NormalizedRange) Start() int {
Severity: Minor
Found in normalizedstring/range.go by golint

exported function NewDefault should have comment or be unexported
Open

func NewDefault() *BPEModel {
Severity: Minor
Found in models/bpemodel/bpemodel.go by golint

exported method BPEModel.Tokenize should have comment or be unexported
Open

func (m *BPEModel) Tokenize(sequence string) ([]models.Token, error) {
Severity: Minor
Found in models/bpemodel/bpemodel.go by golint

exported function NewOriginalRange should have comment or be unexported
Open

func NewOriginalRange(start, end int) OriginalRange {
Severity: Minor
Found in normalizedstring/range.go by golint

exported type NormalizedByteSplit should have comment or be unexported
Open

type NormalizedByteSplit struct {
Severity: Minor
Found in pretokenizedstring/split.go by golint

exported method RegexpSplitPattern.FindMatches should have comment or be unexported
Open

func (sp *RegexpSplitPattern) FindMatches(s string) ([]Capture, error) {
Severity: Minor
Found in splitpattern/regexp.go by golint

exported function EncodingFromEncodableTokens should have comment or be unexported
Open

func EncodingFromEncodableTokens(tokens []EncodableToken) *Encoding {
Severity: Minor
Found in encodings/encodings.go by golint

exported method PreTokenizedString.Splits should have comment or be unexported
Open

func (p *PreTokenizedString) Splits() []Split {
Severity: Minor
Found in pretokenizedstring/pretokenizedstring.go by golint

exported type OriginalByteSplit should have comment or be unexported
Open

type OriginalByteSplit struct {
Severity: Minor
Found in pretokenizedstring/split.go by golint
Severity
Category
Status
Source
Language