com.yahoo.prelude.fastsearch

Class VespaBackEndSearcher

java.lang.Object

com.yahoo.component.AbstractComponent

com.yahoo.component.chain.ChainedComponent

com.yahoo.processing.Processor

com.yahoo.search.Searcher

com.yahoo.search.cluster.PingableSearcher

com.yahoo.prelude.fastsearch.VespaBackEndSearcher

All Implemented Interfaces:

com.yahoo.component.Component, Comparable<com.yahoo.component.Component>

Direct Known Subclasses:

FastSearcher, VdsStreamingSearcher

public abstract class VespaBackEndSearcher
extends PingableSearcher

Superclass for backend searchers.

Author:

baldersheim

Field Summary

Fields

Modifier and Type	Field and Description
protected CacheControl	cacheControl Cache wrapper
private static com.yahoo.processing.request.CompoundName	combinerows
private String	defaultDocsumClass Default docsum class.
private DocumentDatabase	defaultDocumentDb
private static com.yahoo.processing.request.CompoundName	dispatchSummaries If this is turned on this will fill summaries by dispatching directly to search nodes over RPC
private Map<String,DocumentDatabase>	documentDbs The set of all document databases available in the backend handled by this searcher
private static com.yahoo.processing.request.CompoundName	grouping
private boolean	localDispatching
private String	name The name of this source
protected static com.yahoo.processing.request.CompoundName	PACKET_COMPRESSION_LIMIT
protected static com.yahoo.processing.request.CompoundName	PACKET_COMPRESSION_TYPE
private int	rowBits The number of last significant bits in the partId which specifies the row number in this backend, the rest specifies the column.
private int	sourceNumber Searchcluster number
protected static com.yahoo.processing.request.CompoundName	TRACE_DISABLE

Fields inherited from class com.yahoo.component.AbstractComponent

isDeconstructable

Constructor Summary

Constructors

Constructor and Description
VespaBackEndSearcher()

Method Summary

Modifier and Type	Method and Description
private boolean	addCachedHits(Result result, PacketWrapper packetWrapper, String summaryClass, List<DocumentInfo> documents)
protected void	addMetaInfo(Query query, QueryPacketData queryPacketData, QueryResultPacket resultPacket, Result result, boolean fromCache)
protected boolean	addUnfilledHits(Result result, List<DocumentInfo> documents, boolean fromCache, QueryPacketData queryPacketData, CacheKey cacheKey) Creates unfilled hits from a List of DocumentInfo instances.
private Result	cacheLookupFirstPhase(CacheKey key, QueryPacketData queryPacketData, Query query, int offset, int hits, String summaryClass)
protected PacketWrapper	cacheLookupTwoPhase(CacheKey cacheKey, Result result, String summaryClass)
private void	decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata)
private void	decodeSummary(String summaryClass, FastHit hit, byte[] docsumdata, DocsumDefinitionSet docsumSet)
protected abstract void	doPartialFill(Result result, String summaryClass)
protected abstract Result	doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) Searches a search cluster This is an endpoint - searchers will never propagate the search to any nested searcher.
protected static void	ensureInstanceOf(Class<? extends BasicPacket> type, BasicPacket packet, String name) Throws an IOException if the packet is not of the expected type
private void	extractDocumentInfo(FastHit hit, DocumentInfo document)
void	fill(Result result, String summaryClass, Execution execution) Fill hit properties with data using the given summary class.
private boolean	fillHit(FastHit hit, DocsumPacket packet, String summaryClass)
protected int	fillHits(Result result, int packetIndex, Packet[] packets, String summaryClass) Fills the hits.
CacheControl	getCacheControl() Returns the packet cache controller of this
private Result	getCached(CacheKey cacheKey, QueryPacketData queryPacketData, Query query) Returns a cached result, or null if no result was cached for this key
protected String	getDefaultDocsumClass()
protected DocsumDefinitionSet	getDocsumDefinitionSet(Query query)
protected DocumentDatabase	getDocumentDatabase(Query query)
protected String	getName()
static VespaBackEndSearcher	getSearcher(String s)
protected Iterator<Hit>	hitIterator(Result result) Returns an iterator which returns all hits below this result
void	init(SummaryParameters docSumParams, ClusterParams clusterParams, CacheParams cacheParams, DocumentdbInfoConfig documentdbInfoConfig)
boolean	isLocalDispatching()
protected boolean	isLoggingFine()
private void	mergeErrorsInto(Result destination, Result source)
private List<Result>	partitionHits(Result result, String summaryClass)
private void	resolveDocumentDatabase(Query query)
Result	search(Query query, Execution execution) Override this to implement your searcher.
private void	setDefaultDocsumClass(String docsumClass) Sets default document summary class.
void	setLocalDispatching(boolean localDispatching)
protected boolean	summaryNeedsQuery(Query query) Returns whether we need to send the query when fetching summaries.
(package private) static void	traceQuery(String sourceName, String type, Query query, int offset, int hits, int level, Optional<String> quotedSummaryClass)
protected void	transformQuery(Query query)
private boolean	useRowInIndexUri(Result result)
protected static boolean	wantsRPCSummaryFill(Query query)

Methods inherited from class com.yahoo.search.cluster.PingableSearcher

ping

Methods inherited from class com.yahoo.search.Searcher

ensureFilled, getLogger, process, toString

Methods inherited from class com.yahoo.component.chain.ChainedComponent

getAnnotatedDependencies, getDefaultAnnotatedDependencies, getDependencies, initDependencies

Methods inherited from class com.yahoo.component.AbstractComponent

clone, compareTo, deconstruct, getClassName, getId, getIdString, hasInitializedId, initId, isDeconstructable, setIsDeconstructable

Methods inherited from class java.lang.Object

equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

grouping

private static final com.yahoo.processing.request.CompoundName grouping

combinerows

private static final com.yahoo.processing.request.CompoundName combinerows

dispatchSummaries

private static final com.yahoo.processing.request.CompoundName dispatchSummaries

If this is turned on this will fill summaries by dispatching directly to search nodes over RPC

PACKET_COMPRESSION_LIMIT

protected static final com.yahoo.processing.request.CompoundName PACKET_COMPRESSION_LIMIT

PACKET_COMPRESSION_TYPE

protected static final com.yahoo.processing.request.CompoundName PACKET_COMPRESSION_TYPE

TRACE_DISABLE

protected static final com.yahoo.processing.request.CompoundName TRACE_DISABLE

documentDbs

private Map<String,DocumentDatabase> documentDbs

The set of all document databases available in the backend handled by this searcher

defaultDocumentDb

private DocumentDatabase defaultDocumentDb

defaultDocsumClass

private String defaultDocsumClass

Default docsum class. null means "unset" and is the default value

localDispatching

private boolean localDispatching

name

private String name

The name of this source

cacheControl

protected CacheControl cacheControl

Cache wrapper

rowBits

private int rowBits

The number of last significant bits in the partId which specifies the row number in this backend, the rest specifies the column. 0 if not known.

sourceNumber

private int sourceNumber

Searchcluster number

Constructor Detail

VespaBackEndSearcher

public VespaBackEndSearcher()

Method Detail

hitIterator

protected Iterator<Hit> hitIterator(Result result)

Returns an iterator which returns all hits below this result

getName

protected final String getName()

getDefaultDocsumClass

protected final String getDefaultDocsumClass()

setDefaultDocsumClass

private void setDefaultDocsumClass(String docsumClass)

Sets default document summary class. Default is null

getCacheControl

public final CacheControl getCacheControl()

Returns the packet cache controller of this

doSearch2

protected abstract Result doSearch2(Query query,
                                    QueryPacket queryPacket,
                                    CacheKey cacheKey,
                                    Execution execution)

Searches a search cluster This is an endpoint - searchers will never propagate the search to any nested searcher.

Parameters:

query - the query to search

queryPacket - the serialized query representation to pass to the search cluster

cacheKey - the cache key created from the query packet, or null if caching is not used

execution - the query execution context

doPartialFill

protected abstract void doPartialFill(Result result,
                                      String summaryClass)

wantsRPCSummaryFill

protected static boolean wantsRPCSummaryFill(Query query)

summaryNeedsQuery

protected boolean summaryNeedsQuery(Query query)

Returns whether we need to send the query when fetching summaries. This is necessary if the query requests summary features or dynamic snippeting

cacheLookupFirstPhase

private Result cacheLookupFirstPhase(CacheKey key,
                                     QueryPacketData queryPacketData,
                                     Query query,
                                     int offset,
                                     int hits,
                                     String summaryClass)
                              throws IOException

Throws:

IOException

getDocumentDatabase

protected DocumentDatabase getDocumentDatabase(Query query)

resolveDocumentDatabase

private void resolveDocumentDatabase(Query query)

init

public final void init(SummaryParameters docSumParams,
                       ClusterParams clusterParams,
                       CacheParams cacheParams,
                       DocumentdbInfoConfig documentdbInfoConfig)

transformQuery

protected void transformQuery(Query query)

search

public Result search(Query query,
                     Execution execution)

Description copied from class: Searcher

Override this to implement your searcher.

Searcher implementation subclasses will, depending on their type of logic, do one of the following:

• Query processors: Access the query, then call execution.search and return the result
• Result processors: Call execution.search to get the result, access it and return
• Sources (which produces results): Create a result, add the desired hits and return it.
• Federators (which forwards the search to multiple subchains): Call search on the desired subchains in parallel and get the results. Combine the results to one and return it.
• Workflows: Call execution.search as many times as desired, using different queries. Eventually return a result.

Hits come in two kinds - concrete hits are actual content of the kind requested by the user, meta hits are hits which provides information about the collection of hits, on the query, the service and so on.

The query specifies a window into a larger result list that must be returned from the searcher through hits and offset; Searchers which returns list of hits in the top level in the result must return at least hits number of hits (or if impossible; all that are available), starting at the given offset. In addition, searchers are allowed to return any number of meta hits (although this number is expected to be low). For hits contained in nested hit groups, the concept of a window defined by hits and offset is not well defined and does not apply.

Error handling in searchers:

• Unexpected events: Throw any RuntimeException. This query will fail with the exception message, and the error will be logged
• Expected events: Create (new Result(Query, ErrorMessage) or add result.setErrorIfNoOtherErrors(ErrorMessage) an error message to the Result.
• Recoverable user errors: Add a FeedbackHit explaining the condition and how to correct it.

Specified by:

search in class Searcher

Parameters:

query - the query

Returns:

the result of making this query

getCached

private Result getCached(CacheKey cacheKey,
                         QueryPacketData queryPacketData,
                         Query query)

Returns a cached result, or null if no result was cached for this key

Parameters:

cacheKey - the cache key created from the query packet

queryPacketData - a serialization of the query, to avoid having to recompute this, or null if not available

query - the query, used for tracing, lookup of result window and result creation

partitionHits

private List<Result> partitionHits(Result result,
                                   String summaryClass)

fill

public void fill(Result result,
                 String summaryClass,
                 Execution execution)

Description copied from class: Searcher

Fill hit properties with data using the given summary class. Calling this on already filled results has no cost.

This needs to be overridden by federating searchers to contact search sources again by propagating the fill call down through the search chain, and by source searchers which talks to fill capable backends to request the data to be filled. Other searchers do not need to override this.

Overrides:

fill in class Searcher

Parameters:

result - the result to fill

summaryClass - the name of the collection of fields to fetch the values of, or null to use the default

mergeErrorsInto

private void mergeErrorsInto(Result destination,
                             Result source)

traceQuery

static void traceQuery(String sourceName,
                       String type,
                       Query query,
                       int offset,
                       int hits,
                       int level,
                       Optional<String> quotedSummaryClass)

addMetaInfo

protected void addMetaInfo(Query query,
                           QueryPacketData queryPacketData,
                           QueryResultPacket resultPacket,
                           Result result,
                           boolean fromCache)

fillHit

private boolean fillHit(FastHit hit,
                        DocsumPacket packet,
                        String summaryClass)

fillHits

protected int fillHits(Result result,
                       int packetIndex,
                       Packet[] packets,
                       String summaryClass)
                throws IOException

Fills the hits.

Returns:

the number of hits that we did not return data for, i.e when things are working normally we return 0.

Throws:

IOException

ensureInstanceOf

protected static void ensureInstanceOf(Class<? extends BasicPacket> type,
                                       BasicPacket packet,
                                       String name)
                                throws IOException

Throws an IOException if the packet is not of the expected type

Throws:

IOException

addCachedHits

private boolean addCachedHits(Result result,
                              PacketWrapper packetWrapper,
                              String summaryClass,
                              List<DocumentInfo> documents)

useRowInIndexUri

private boolean useRowInIndexUri(Result result)

extractDocumentInfo

private void extractDocumentInfo(FastHit hit,
                                 DocumentInfo document)

cacheLookupTwoPhase

protected PacketWrapper cacheLookupTwoPhase(CacheKey cacheKey,
                                            Result result,
                                            String summaryClass)

getDocsumDefinitionSet

protected DocsumDefinitionSet getDocsumDefinitionSet(Query query)

decodeSummary

private void decodeSummary(String summaryClass,
                           FastHit hit,
                           byte[] docsumdata)

decodeSummary

private void decodeSummary(String summaryClass,
                           FastHit hit,
                           byte[] docsumdata,
                           DocsumDefinitionSet docsumSet)

addUnfilledHits

protected boolean addUnfilledHits(Result result,
                                  List<DocumentInfo> documents,
                                  boolean fromCache,
                                  QueryPacketData queryPacketData,
                                  CacheKey cacheKey)

Creates unfilled hits from a List of DocumentInfo instances. Do note cacheKey should be available if a cache is active, even if the hit is not created from a cache in the current call path.

Parameters:

queryPacketData - binary data from first phase of search, or null

cacheKey - the key this hit should match in the packet cache, or null

getSearcher

public static VespaBackEndSearcher getSearcher(String s)

isLoggingFine

protected boolean isLoggingFine()

isLocalDispatching

public boolean isLocalDispatching()

setLocalDispatching

public void setLocalDispatching(boolean localDispatching)