com
.
johnsnowlabs
.
nlp
.
annotators
.
tokenizer
bpe
package
bpe
Visibility
Public
All
Type Members
class
RobertaTokenizer
extends
BpeTokenizer
case class
SpecialToken
(
content:
String
,
id:
Int
,
singleWord:
Boolean
=
false
,
lstrip:
Boolean
=
false
,
rstrip:
Boolean
=
false
)
extends
Product
with
Serializable
Value Members
object
BpeTokenizer
Ungrouped