| .codeclimate.json | |
|
|
| .github/workflows/go-ci.yml | |
|
|
| .gitignore | |
|
|
| LICENSE | |
|
|
| README.md | |
|
|
| encodings/encodings.go | 74 | A 1 hr |
|
| go.mod | |
|
|
| go.sum | |
|
|
| gotokenizers.go | 1 | A 0 mins |
|
| models/bpemodel/bpemodel.go | 166 | A 2 hrs |
|
| models/bpemodel/mergemap.go | 71 | A 45 mins |
|
| models/bpemodel/testdata/merges.txt | |
|
|
| models/bpemodel/word.go | 113 | A 3 hrs |
|
| models/bpemodel/wordcache.go | 27 | A 0 mins |
|
| models/bpemodel/wordmerge.go | 29 | A 0 mins |
|
| models/models.go | 10 | A 0 mins |
|
| models/wordpiecemodel/wordpiecemodel.go | 95 | A 2 hrs |
|
| normalizedstring/normalizedstring.go | 568 | D 1 day |
|
| normalizedstring/range.go | 40 | A 0 mins |
|
| normalizers/bertnormalizer/bertnormalizer.go | 99 | A 0 mins |
|
| normalizers/lowercasenormalizer/lowercasenormalizer.go | 14 | A 0 mins |
|
| normalizers/normalizers.go | 5 | A 0 mins |
|
| normalizers/sequencenormalizer/sequencenormalizer.go | 19 | A 0 mins |
|
| normalizers/stripnormalizer/stripnormalizer.go | 14 | A 0 mins |
|
| pretokenizedstring/pretokenizedstring.go | 156 | A 0 mins |
|
| pretokenizedstring/split.go | 27 | A 0 mins |
|
| pretokenizers/bertpretokenizer/bertpretokenizer.go | 42 | A 40 mins |
|
| pretokenizers/bytelevelpretokenizer/bytelevelpretokenizer.go | 87 | A 40 mins |
|
| pretokenizers/metaspacepretokenizer/metaspacepretokenizer.go | 44 | A 0 mins |
|
| pretokenizers/pretokenizers.go | 10 | A 0 mins |
|
| pretokenizers/runedelimiterpretokenizer/runedelimiterpretokenizer.go | 26 | A 0 mins |
|
| pretokenizers/whitespacepretokenizer/whitespacepretokenizer.go | 31 | A 0 mins |
|
| pretokenizers/whitespacesplitpretokenizer/whitespacesplitpretokenizer.go | 27 | A 0 mins |
|
| splitpattern/func.go | 45 | A 0 mins |
|
| splitpattern/invertedpattern.go | 18 | A 0 mins |
|
| splitpattern/regexp.go | 45 | A 0 mins |
|
| splitpattern/regexp2.go | 53 | A 0 mins |
|
| splitpattern/rune.go | 17 | A 0 mins |
|
| splitpattern/splitpattern.go | 9 | A 0 mins |
|
| splitpattern/string.go | 26 | A 0 mins |
|
| strutils/strutils.go | 43 | A 0 mins |
|
| vocabulary/testdata/vocab.json | |
|
|
| vocabulary/vocabulary.go | 50 | A 0 mins |
|