com.yahoo.vespa.hadoop.pig

Class VespaStorage

java.lang.Object

org.apache.pig.StoreFunc

com.yahoo.vespa.hadoop.pig.VespaStorage

All Implemented Interfaces:

org.apache.pig.StoreFuncInterface

public class VespaStorage
extends org.apache.pig.StoreFunc

A small Pig UDF wrapper around the Vespa http client for feeding data into a Vespa endpoint.

Author:

lesters

Field Summary

Fields

Modifier and Type	Field and Description
private boolean	createDocOp
private VespaDocumentOperation.Operation	operation
(package private) Properties	properties
private static String	PROPERTY_CREATE_DOC_OP
private static String	PROPERTY_ID_TEMPLATE
private static String	PROPERTY_OPERATION
private static String	PROPERTY_RESOURCE_SCHEMA
private org.apache.hadoop.mapreduce.RecordWriter	recordWriter
private org.apache.pig.ResourceSchema	resourceSchema
private String	signature
private String	template

Constructor Summary

Constructors

Constructor and Description
VespaStorage()
VespaStorage(String... params)

Method Summary

Modifier and Type	Method and Description
static <T extends Serializable> T	base64Deserialize(String s)
static String	base64Serialize(Object o)
void	checkSchema(org.apache.pig.ResourceSchema resourceSchema)
void	cleanupOnFailure(String s, org.apache.hadoop.mapreduce.Job job)
void	cleanupOnSuccess(String s, org.apache.hadoop.mapreduce.Job job)
private String	createDocumentOperation(org.apache.pig.data.Tuple tuple)
org.apache.hadoop.mapreduce.OutputFormat	getOutputFormat()
private org.apache.pig.ResourceSchema	getResourceSchema()
private Properties	getUDFProperties()
void	prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter recordWriter)
void	putNext(org.apache.pig.data.Tuple tuple)
String	relToAbsPathForStoreLocation(String endpoint, org.apache.hadoop.fs.Path path)
private void	setResourceSchema(org.apache.pig.ResourceSchema schema)
void	setStoreFuncUDFContextSignature(String s)
void	setStoreLocation(String endpoint, org.apache.hadoop.mapreduce.Job job)

Methods inherited from class org.apache.pig.StoreFunc

cleanupOnFailureImpl, warn

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

createDocOp

private final boolean createDocOp

template

private final String template

operation

private final VespaDocumentOperation.Operation operation

signature

private String signature

recordWriter

private org.apache.hadoop.mapreduce.RecordWriter recordWriter

resourceSchema

private org.apache.pig.ResourceSchema resourceSchema

PROPERTY_CREATE_DOC_OP

private static final String PROPERTY_CREATE_DOC_OP

See Also:

Constant Field Values

PROPERTY_ID_TEMPLATE

private static final String PROPERTY_ID_TEMPLATE

See Also:

Constant Field Values

PROPERTY_OPERATION

private static final String PROPERTY_OPERATION

See Also:

Constant Field Values

PROPERTY_RESOURCE_SCHEMA

private static final String PROPERTY_RESOURCE_SCHEMA

See Also:

Constant Field Values

properties

Properties properties

Constructor Detail

VespaStorage

public VespaStorage()

VespaStorage

public VespaStorage(String... params)

Method Detail

getOutputFormat

public org.apache.hadoop.mapreduce.OutputFormat getOutputFormat()
                                                         throws IOException

Specified by:

getOutputFormat in interface org.apache.pig.StoreFuncInterface

Specified by:

getOutputFormat in class org.apache.pig.StoreFunc

Throws:

IOException

setStoreLocation

public void setStoreLocation(String endpoint,
                             org.apache.hadoop.mapreduce.Job job)
                      throws IOException

Specified by:

setStoreLocation in interface org.apache.pig.StoreFuncInterface

Specified by:

setStoreLocation in class org.apache.pig.StoreFunc

Throws:

IOException

prepareToWrite

public void prepareToWrite(org.apache.hadoop.mapreduce.RecordWriter recordWriter)
                    throws IOException

Specified by:

prepareToWrite in interface org.apache.pig.StoreFuncInterface

Specified by:

prepareToWrite in class org.apache.pig.StoreFunc

Throws:

IOException

putNext

public void putNext(org.apache.pig.data.Tuple tuple)
             throws IOException

Specified by:

putNext in interface org.apache.pig.StoreFuncInterface

Specified by:

putNext in class org.apache.pig.StoreFunc

Throws:

IOException

checkSchema

public void checkSchema(org.apache.pig.ResourceSchema resourceSchema)
                 throws IOException

Specified by:

checkSchema in interface org.apache.pig.StoreFuncInterface

Overrides:

checkSchema in class org.apache.pig.StoreFunc

Throws:

IOException

relToAbsPathForStoreLocation

public String relToAbsPathForStoreLocation(String endpoint,
                                           org.apache.hadoop.fs.Path path)
                                    throws IOException

Specified by:

relToAbsPathForStoreLocation in interface org.apache.pig.StoreFuncInterface

Overrides:

relToAbsPathForStoreLocation in class org.apache.pig.StoreFunc

Throws:

IOException

setStoreFuncUDFContextSignature

public void setStoreFuncUDFContextSignature(String s)

Specified by:

setStoreFuncUDFContextSignature in interface org.apache.pig.StoreFuncInterface

Overrides:

setStoreFuncUDFContextSignature in class org.apache.pig.StoreFunc

cleanupOnFailure

public void cleanupOnFailure(String s,
                             org.apache.hadoop.mapreduce.Job job)
                      throws IOException

Specified by:

cleanupOnFailure in interface org.apache.pig.StoreFuncInterface

Overrides:

cleanupOnFailure in class org.apache.pig.StoreFunc

Throws:

IOException

cleanupOnSuccess

public void cleanupOnSuccess(String s,
                             org.apache.hadoop.mapreduce.Job job)
                      throws IOException

Specified by:

cleanupOnSuccess in interface org.apache.pig.StoreFuncInterface

Overrides:

cleanupOnSuccess in class org.apache.pig.StoreFunc

Throws:

IOException

getResourceSchema

private org.apache.pig.ResourceSchema getResourceSchema()
                                                 throws IOException

Throws:

IOException

setResourceSchema

private void setResourceSchema(org.apache.pig.ResourceSchema schema)
                        throws IOException

Throws:

IOException

getUDFProperties

private Properties getUDFProperties()

createDocumentOperation

private String createDocumentOperation(org.apache.pig.data.Tuple tuple)
                                throws IOException

Throws:

IOException

base64Serialize

public static String base64Serialize(Object o)
                              throws IOException

Throws:

IOException

base64Deserialize

public static <T extends Serializable> T base64Deserialize(String s)
                                                    throws IOException

Throws:

IOException