Package ai.djl.sentencepiece.jni

Class SentencePieceLibrary

java.lang.Object

ai.djl.sentencepiece.jni.SentencePieceLibrary

public final class SentencePieceLibrary
extends java.lang.Object

A class containing utilities to interact with the SentencePiece Engine's JNI layer.

Field Summary

Fields

Modifier and Type	Field	Description
static SentencePieceLibrary	LIB

Method Summary

Modifier and Type	Method	Description
long	createSentencePieceProcessor()
java.lang.String	decode(long handle, int[] ids)
void	deleteSentencePieceProcessor(long handle)
java.lang.String	detokenize(long handle, java.lang.String[] tokens)
int[]	encode(long handle, java.lang.String text)
java.lang.String	idToPiece(long handle, int id)
void	loadModel(long handle, java.lang.String filePath)
void	loadModelFromBytes(long handle, byte[] bytes)
int	pieceToId(long handle, java.lang.String piece)
java.lang.String[]	tokenize(long handle, java.lang.String text)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LIB

public static final SentencePieceLibrary LIB

Method Detail

createSentencePieceProcessor

public long createSentencePieceProcessor()

loadModel

public void loadModel(long handle,
                      java.lang.String filePath)

loadModelFromBytes

public void loadModelFromBytes(long handle,
                               byte[] bytes)

deleteSentencePieceProcessor

public void deleteSentencePieceProcessor(long handle)

tokenize

public java.lang.String[] tokenize(long handle,
                                   java.lang.String text)

encode

public int[] encode(long handle,
                    java.lang.String text)

detokenize

public java.lang.String detokenize(long handle,
                                   java.lang.String[] tokens)

decode

public java.lang.String decode(long handle,
                               int[] ids)

idToPiece

public java.lang.String idToPiece(long handle,
                                  int id)

pieceToId

public int pieceToId(long handle,
                     java.lang.String piece)