org.apache.drill.exec.planner.index

Class IndexPlanUtils

java.lang.Object

org.apache.drill.exec.planner.index.IndexPlanUtils

public class IndexPlanUtils
extends Object

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	IndexPlanUtils.ConditionIndexed

Constructor Summary

Constructors

Constructor and Description
IndexPlanUtils()

Method Summary

Modifier and Type	Method and Description
static org.apache.calcite.rel.RelCollation	buildCollationCoveringIndexScan(IndexDescriptor indexDesc, IndexCallContext context) Build the collation property for index scan
static org.apache.calcite.rel.RelCollation	buildCollationForExpressions(Map<LogicalExpression,Integer> projectExprs, IndexDescriptor indexDesc, IndexCallContext context) Given index, compute the collations for a list of projected expressions(from Scan's rowType or Project's ) in the context
static org.apache.calcite.rel.RelCollation	buildCollationLowerProject(List<org.apache.calcite.rex.RexNode> projectRexs, org.apache.calcite.rel.RelNode input, FunctionalIndexInfo indexInfo) Build collation property for the 'lower' project, the one closer to the Scan
static org.apache.calcite.rel.RelCollation	buildCollationNonCoveringIndexScan(IndexDescriptor indexDesc, org.apache.calcite.rel.type.RelDataType indexScanRowType, org.apache.calcite.rel.type.RelDataType restrictedScanRowType, IndexCallContext context)
static org.apache.calcite.rel.RelCollation	buildCollationProject(List<org.apache.calcite.rex.RexNode> projectRexs, DrillProjectRelBase project, org.apache.calcite.rel.RelNode input, FunctionalIndexInfo indexInfo, IndexCallContext context) Build collation property for project, the one closer to the Scan
static org.apache.calcite.rel.RelCollation	buildCollationUpperProject(List<org.apache.calcite.rex.RexNode> projectRexs, org.apache.calcite.rel.RelCollation inputCollation, FunctionalIndexInfo indexInfo, Map<Integer,List<org.apache.calcite.rex.RexNode>> collationFilterMap) Build collation property for the 'upper' project, the one above the filter
static ScanPrel	buildCoveringIndexScan(DrillScanRelBase origScan, IndexGroupScan indexGroupScan, IndexCallContext indexContext, IndexDescriptor indexDesc)
static boolean	checkScan(DrillScanRel scanRel) check if we want to apply index rules on this scan, if group scan is not instance of DbGroupScan, or this DbGroupScan instance does not support secondary index, or this scan is already an index scan or Restricted Scan, do not apply index plan rules on it.
static IndexPlanUtils.ConditionIndexed	conditionIndexed(IndexableExprMarker exprMarker, IndexDescriptor indexDesc) Check if any of the fields of the index are present in a list of LogicalExpressions supplied as part of IndexableExprMarker
static boolean	generateLimit(OrderedRel sort)
static org.apache.calcite.rel.RelCollation	getCollation(org.apache.calcite.rel.core.Sort sort)
static List<DrillDistributionTrait.DistributionField>	getDistributionField(org.apache.calcite.rel.core.Sort rel)
static Map<LogicalExpression,Integer>	getExprsFromRowType(org.apache.calcite.rel.type.RelDataType indexScanRowType)
static org.apache.calcite.rex.RexNode	getFetch(OrderedRel sort)
static GroupScan	getGroupScan(DrillScanRelBase relNode)
static int	getIndexFromCollation(int refIndex, List<org.apache.calcite.rel.RelFieldCollation> inputFieldCollations)
static org.apache.calcite.rex.RexNode	getLeadingColumnsFilter(List<org.apache.calcite.rex.RexNode> leadingFilters, org.apache.calcite.rex.RexBuilder rexBuilder)
static List<org.apache.calcite.rex.RexNode>	getLeadingFilters(Map<LogicalExpression,org.apache.calcite.rex.RexNode> leadingPrefixMap, List<LogicalExpression> indexCols)
static org.apache.calcite.rex.RexNode	getLeadingPrefixMap(Map<LogicalExpression,org.apache.calcite.rex.RexNode> leadingPrefixMap, List<LogicalExpression> indexCols, IndexConditionInfo.Builder builder, org.apache.calcite.rex.RexNode condition)
static org.apache.calcite.rex.RexNode	getOffset(OrderedRel sort)
static Map<LogicalExpression,Integer>	getProjectExprs(List<org.apache.calcite.rex.RexNode> projectRexs, DrillProjectRelBase project, org.apache.calcite.rel.RelNode input)
static List<org.apache.calcite.rex.RexNode>	getProjects(DrillProjectRelBase proj)
static org.apache.calcite.rex.RexNode	getTotalFilter(org.apache.calcite.rex.RexNode leadColsFilter, org.apache.calcite.rex.RexNode totRemColsFilter, org.apache.calcite.rex.RexBuilder rexBuilder)
static org.apache.calcite.rex.RexNode	getTotalRemainderFilter(org.apache.calcite.rex.RexNode indexColsRemFilter, org.apache.calcite.rex.RexNode incColsRemFilter, org.apache.calcite.rex.RexBuilder rexBuilder)
static boolean	isCoveringIndex(IndexCallContext indexContext, FunctionalIndexInfo functionInfo) For a particular table scan for table T1 and an index on that table, find out if it is a covering index
static boolean	pathOnlyInIndexedFunction(SchemaPath path)
static org.apache.calcite.rex.RexNode	rewriteFunctionalRex(IndexCallContext indexContext, DrillParseContext parseContext, DrillProjectRelBase project, org.apache.calcite.rel.RelNode scan, org.apache.calcite.rex.RexNode toRewriteRex, org.apache.calcite.rel.type.RelDataType newRowType, FunctionalIndexInfo functionInfo) A RexNode forest with three RexNodes for expressions "cast(a.q as int) * 2, b+c, concat(a.q, " world")" on Scan RowType('a', 'b', 'c') will be like this: (0)Call:"*" Call:"concat" / \ / \ (1)Call:CAST 2 Call:"+" (5)Call:ITEM ' world' / \ / \ / \ (2)Call:ITEM TYPE:INT (3)$1 (4)$2 $0 'q' / \ $0 'q' So for above expressions, when visiting the RexNode trees using PathInExpr, we could mark indexed expressions in the trees, as shown in the diagram above are the node (1), then collect the schema paths in the indexed expression but found out of the indexed expression -- node (5), and other regular schema paths (3) (4)
static List<SchemaPath>	rewriteFunctionColumn(List<SchemaPath> paths, FunctionalIndexInfo functionInfo, List<SchemaPath> addedPaths) For IndexGroupScan, if a column is only appeared in the should-be-renamed function, this column is to-be-replaced column, we replace that column(schemaPath) from 'a.b' to '$1' in the list of SchemaPath.
static boolean	scanIsPartition(GroupScan scan)
static void	updateSortExpression(IndexCallContext indexContext, List<org.apache.calcite.rel.RelFieldCollation> coll) generate logical expressions for sort rexNodes in SortRel, the result is store to IndexPlanCallContext
static void	updateSortExpression(IndexPhysicalPlanCallContext indexContext, List<org.apache.calcite.rel.RelFieldCollation> coll) generate logical expressions for sort rexNodes in SortRel, the result is store to IndexPlanCallContext

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

IndexPlanUtils

public IndexPlanUtils()

Method Detail

conditionIndexed

public static IndexPlanUtils.ConditionIndexed conditionIndexed(IndexableExprMarker exprMarker,
                                                               IndexDescriptor indexDesc)

Check if any of the fields of the index are present in a list of LogicalExpressions supplied as part of IndexableExprMarker

Parameters:

exprMarker - the marker that has analyzed original index condition on top of original scan

indexDesc - the index definition plus functions to access materialized index

Returns:

ConditionIndexed.FULL, PARTIAL or NONE depending on whether all, some or no columns of the indexDesc are present in the list of LogicalExpressions supplied as part of exprMarker

checkScan

public static boolean checkScan(DrillScanRel scanRel)

check if we want to apply index rules on this scan, if group scan is not instance of DbGroupScan, or this DbGroupScan instance does not support secondary index, or this scan is already an index scan or Restricted Scan, do not apply index plan rules on it.

Parameters:

scanRel - the current scan rel

Returns:

true to indicate that we want to apply index rules on this scan, otherwise false

isCoveringIndex

public static boolean isCoveringIndex(IndexCallContext indexContext,
                                      FunctionalIndexInfo functionInfo)

For a particular table scan for table T1 and an index on that table, find out if it is a covering index

Returns:

true if it is a covering index, otherwise false

buildCollationLowerProject

public static org.apache.calcite.rel.RelCollation buildCollationLowerProject(List<org.apache.calcite.rex.RexNode> projectRexs,
                                                                             org.apache.calcite.rel.RelNode input,
                                                                             FunctionalIndexInfo indexInfo)

Build collation property for the 'lower' project, the one closer to the Scan

Parameters:

projectRexs - list of row expressions

input - input as a relational expression

indexInfo - collects functional index information

Returns:

the output RelCollation

buildCollationUpperProject

public static org.apache.calcite.rel.RelCollation buildCollationUpperProject(List<org.apache.calcite.rex.RexNode> projectRexs,
                                                                             org.apache.calcite.rel.RelCollation inputCollation,
                                                                             FunctionalIndexInfo indexInfo,
                                                                             Map<Integer,List<org.apache.calcite.rex.RexNode>> collationFilterMap)

Build collation property for the 'upper' project, the one above the filter

Parameters:

projectRexs - list of row expressions

inputCollation - the input collation

indexInfo - collects functional index information

collationFilterMap - map for collation filter

Returns:

the output RelCollation

getIndexFromCollation

public static int getIndexFromCollation(int refIndex,
                                        List<org.apache.calcite.rel.RelFieldCollation> inputFieldCollations)

getProjects

public static List<org.apache.calcite.rex.RexNode> getProjects(DrillProjectRelBase proj)

generateLimit

public static boolean generateLimit(OrderedRel sort)

getOffset

public static org.apache.calcite.rex.RexNode getOffset(OrderedRel sort)

getFetch

public static org.apache.calcite.rex.RexNode getFetch(OrderedRel sort)

updateSortExpression

public static void updateSortExpression(IndexCallContext indexContext,
                                        List<org.apache.calcite.rel.RelFieldCollation> coll)

generate logical expressions for sort rexNodes in SortRel, the result is store to IndexPlanCallContext

Parameters:

indexContext - the index call context

coll - list of field collations

updateSortExpression

public static void updateSortExpression(IndexPhysicalPlanCallContext indexContext,
                                        List<org.apache.calcite.rel.RelFieldCollation> coll)

generate logical expressions for sort rexNodes in SortRel, the result is store to IndexPlanCallContext

Parameters:

indexContext - the index call context

coll - list of field collations

buildCollationProject

public static org.apache.calcite.rel.RelCollation buildCollationProject(List<org.apache.calcite.rex.RexNode> projectRexs,
                                                                        DrillProjectRelBase project,
                                                                        org.apache.calcite.rel.RelNode input,
                                                                        FunctionalIndexInfo indexInfo,
                                                                        IndexCallContext context)

Build collation property for project, the one closer to the Scan

Parameters:

projectRexs - the expressions to project

project - the project between projectRexs and input, it could be null if no such intermediate project(lower project)

input - the input RelNode to the project, usually it is the scan operator.

indexInfo - the index for which we are building index plan

context - the context of this index planning process

Returns:

the output RelCollation

buildCollationCoveringIndexScan

public static org.apache.calcite.rel.RelCollation buildCollationCoveringIndexScan(IndexDescriptor indexDesc,
                                                                                  IndexCallContext context)

Build the collation property for index scan

Parameters:

indexDesc - the index for which we are building index plan

context - the context of this index planning process

Returns:

the output RelCollation for the scan on index

getProjectExprs

public static Map<LogicalExpression,Integer> getProjectExprs(List<org.apache.calcite.rex.RexNode> projectRexs,
                                                             DrillProjectRelBase project,
                                                             org.apache.calcite.rel.RelNode input)

getExprsFromRowType

public static Map<LogicalExpression,Integer> getExprsFromRowType(org.apache.calcite.rel.type.RelDataType indexScanRowType)

buildCollationForExpressions

public static org.apache.calcite.rel.RelCollation buildCollationForExpressions(Map<LogicalExpression,Integer> projectExprs,
                                                                               IndexDescriptor indexDesc,
                                                                               IndexCallContext context)

Given index, compute the collations for a list of projected expressions(from Scan's rowType or Project's ) in the context

Parameters:

projectExprs - the output expression list of a RelNode

indexDesc - the index for which we are building index plan

context - the context of this index planning process

Returns:

the collation provided by index that will be exposed by the expression list

pathOnlyInIndexedFunction

public static boolean pathOnlyInIndexedFunction(SchemaPath path)

buildCollationNonCoveringIndexScan

public static org.apache.calcite.rel.RelCollation buildCollationNonCoveringIndexScan(IndexDescriptor indexDesc,
                                                                                     org.apache.calcite.rel.type.RelDataType indexScanRowType,
                                                                                     org.apache.calcite.rel.type.RelDataType restrictedScanRowType,
                                                                                     IndexCallContext context)

scanIsPartition

public static boolean scanIsPartition(GroupScan scan)

getGroupScan

public static GroupScan getGroupScan(DrillScanRelBase relNode)

getCollation

public static org.apache.calcite.rel.RelCollation getCollation(org.apache.calcite.rel.core.Sort sort)

getDistributionField

public static List<DrillDistributionTrait.DistributionField> getDistributionField(org.apache.calcite.rel.core.Sort rel)

buildCoveringIndexScan

public static ScanPrel buildCoveringIndexScan(DrillScanRelBase origScan,
                                              IndexGroupScan indexGroupScan,
                                              IndexCallContext indexContext,
                                              IndexDescriptor indexDesc)

rewriteFunctionColumn

public static List<SchemaPath> rewriteFunctionColumn(List<SchemaPath> paths,
                                                     FunctionalIndexInfo functionInfo,
                                                     List<SchemaPath> addedPaths)

For IndexGroupScan, if a column is only appeared in the should-be-renamed function, this column is to-be-replaced column, we replace that column(schemaPath) from 'a.b' to '$1' in the list of SchemaPath.

Parameters:

paths - list of paths

functionInfo - functional index information that may impact rewrite

addedPaths - list of paths added

Returns:

list of new paths

rewriteFunctionalRex

public static org.apache.calcite.rex.RexNode rewriteFunctionalRex(IndexCallContext indexContext,
                                                                  DrillParseContext parseContext,
                                                                  DrillProjectRelBase project,
                                                                  org.apache.calcite.rel.RelNode scan,
                                                                  org.apache.calcite.rex.RexNode toRewriteRex,
                                                                  org.apache.calcite.rel.type.RelDataType newRowType,
                                                                  FunctionalIndexInfo functionInfo)

A RexNode forest with three RexNodes for expressions "cast(a.q as int) * 2, b+c, concat(a.q, " world")" on Scan RowType('a', 'b', 'c') will be like this: (0)Call:"*" Call:"concat" / \ / \ (1)Call:CAST 2 Call:"+" (5)Call:ITEM ' world' / \ / \ / \ (2)Call:ITEM TYPE:INT (3)$1 (4)$2 $0 'q' / \ $0 'q' So for above expressions, when visiting the RexNode trees using PathInExpr, we could mark indexed expressions in the trees, as shown in the diagram above are the node (1), then collect the schema paths in the indexed expression but found out of the indexed expression -- node (5), and other regular schema paths (3) (4)

Parameters:

indexContext - the index call context

parseContext - the drill parse context

project - the drill base class for logical and physical projects

scan - a rel node scan

toRewriteRex - the RexNode to be converted if it contains a functional index expression

newRowType - data type for new row

functionInfo - functional index information that may impact rewrite

Returns:

rewritten functional row expression

getLeadingPrefixMap

public static org.apache.calcite.rex.RexNode getLeadingPrefixMap(Map<LogicalExpression,org.apache.calcite.rex.RexNode> leadingPrefixMap,
                                                                 List<LogicalExpression> indexCols,
                                                                 IndexConditionInfo.Builder builder,
                                                                 org.apache.calcite.rex.RexNode condition)

getLeadingFilters

public static List<org.apache.calcite.rex.RexNode> getLeadingFilters(Map<LogicalExpression,org.apache.calcite.rex.RexNode> leadingPrefixMap,
                                                                     List<LogicalExpression> indexCols)

getLeadingColumnsFilter

public static org.apache.calcite.rex.RexNode getLeadingColumnsFilter(List<org.apache.calcite.rex.RexNode> leadingFilters,
                                                                     org.apache.calcite.rex.RexBuilder rexBuilder)

getTotalRemainderFilter

public static org.apache.calcite.rex.RexNode getTotalRemainderFilter(org.apache.calcite.rex.RexNode indexColsRemFilter,
                                                                     org.apache.calcite.rex.RexNode incColsRemFilter,
                                                                     org.apache.calcite.rex.RexBuilder rexBuilder)

getTotalFilter

public static org.apache.calcite.rex.RexNode getTotalFilter(org.apache.calcite.rex.RexNode leadColsFilter,
                                                            org.apache.calcite.rex.RexNode totRemColsFilter,
                                                            org.apache.calcite.rex.RexBuilder rexBuilder)