Impala 源码分析-FE

By yhluo 2015年7月29日 Impala 3 Comments

Impala 源代码目录结构

SQL 解析

Impala 的 SQL 解析与执行计划生成部分是由 impala-frontend（Java）实现的，监听端口是 21000。用户通过
Beeswax 接口 BeeswaxService.query() 提交一个请求，在 impalad 端的处理逻辑是由
void ImpalaServer::query(QueryHandle& query_handle, const Query& query) 这个函数(
ImpalaServer.h)完成的。

1	void ImpalaServer::query(QueryHandle& query_handle, const Query& query) {

2	VLOG_QUERY << "query(): query=" << query.query;

3	ScopedSessionState session_handle(this);

4	shared_ptr<SessionState> session;

5	RAISE_IF_ERROR(// 为当前连接返回唯一标识，标记会话为使用中并保存

6	session_handle.WithSession(ThriftServer::GetThreadConnectionId(), &session),

7	SQLSTATE_GENERAL_ERROR);

8	TQueryCtx query_ctx;

9	// 将 Query 转化为 TQueryCtx

10	// raise general error for request conversion error;

11	RAISE_IF_ERROR(QueryToTQueryContext(query, &query_ctx), SQLSTATE_GENERAL_ERROR);

12

13	// raise Syntax error or access violation; it's likely to be syntax/analysis error

14	// TODO: that may not be true; fix this

15	shared_ptr<QueryExecState> exec_state;

16	// 开始异步执行查询，内部调用 ImpalaServer::Execute() 函数

17	// 将 TQueryCtx 转换为 QueryExecState，注册并调用 Coordinator::Execute()

18	RAISE_IF_ERROR(Execute(&query_ctx, session, &exec_state),

19	SQLSTATE_SYNTAX_ERROR_OR_ACCESS_VIOLATION);

20

21	exec_state->UpdateQueryState(QueryState::RUNNING);

22	// start thread to wait for results to become available, which will allow

23	// us to advance query state to FINISHED or EXCEPTION

24	exec_state->WaitAsync();

25	// Once the query is running do a final check for session closure and add it to the

26	// set of in-flight queries.

27	Status status = SetQueryInflight(session, exec_state);

28	if (!status.ok()) {

29	UnregisterQuery(exec_state->query_id(), false, &status);

30	RaiseBeeswaxException(status.GetDetail(), SQLSTATE_GENERAL_ERROR);

31

}

32	TUniqueIdToQueryHandle(exec_state->query_id(), &query_handle);

33

}

其中 QueryToTQueryContext(query, &query_ctx) 将 Query 装换为 TQueryCtx。具体代码实现如下：
(ImpalaServer.h)

1	Status ImpalaServer::QueryToTQueryContext(const Query& query,

2	TQueryCtx* query_ctx) {

3	query_ctx->request.stmt = query.query;

4	VLOG_QUERY << "query: " << ThriftDebugString(query);

5

{

6	shared_ptr<SessionState> session;

7	const TUniqueId& session_id = ThriftServer::GetThreadConnectionId();

8	RETURN_IF_ERROR(GetSessionState(session_id, &session));

9	DCHECK(session != NULL);

10

{

11	// The session is created when the client connects. Depending on the underlying

12	// transport, the username may be known at that time. If the username hasn't been

13	// set yet, set it now.

14	lock_guard<mutex> l(session->lock);

15	if (session->connected_user.empty()) session->connected_user = query.hadoop_user;

16	query_ctx->request.query_options = session->default_query_options;

17

}

18	// 构建该 SessionState 的 Thrift 表示用于序列化到 frontend

19	session->ToThrift(session_id, &query_ctx->session);

20

}

21

22	// Override default query options with Query.Configuration

23	if (query.__isset.configuration) {

24	BOOST_FOREACH(const string& option, query.configuration) {

25	RETURN_IF_ERROR(ParseQueryOptions(option, &query_ctx->request.query_options));

26

}

27	VLOG_QUERY << "TClientRequest.queryOptions: "

28	<< ThriftDebugString(query_ctx->request.query_options);

29

}

30

31	return Status::OK();

32

}

内部调用 ImpalaServer::Execute()
(ImpalaServer.h)
函数将 TQueryCtx 转换为 TExecRequest，具体逻辑通过调用 ImpalaServer::ExecuteInternal() 实现。代码如下：

1	Status ImpalaServer::Execute(TQueryCtx* query_ctx,

2	shared_ptr<SessionState> session_state,

3	shared_ptr<QueryExecState>* exec_state) {

4	PrepareQueryContext(query_ctx);

5	bool registered_exec_state;

6	ImpaladMetrics::IMPALA_SERVER_NUM_QUERIES->Increment(1L);

7

8	// Redact the SQL stmt and update the query context

9	string stmt = replace_all_copy(query_ctx->request.stmt, " ", " ");

10	Redact(&stmt);

11	query_ctx->request.__set_redacted_stmt((const string) stmt);

12	// 实现 Execute() 逻辑，出错时不取消注册查询

13	Status status = ExecuteInternal(*query_ctx, session_state, &registered_exec_state,

14	exec_state);

15	if (!status.ok() && registered_exec_state) {

16	UnregisterQuery((*exec_state)->query_id(), false, &status);

17

}

18	return status;

19

}

上面的函数调用 ImpalaServer::ExecuteInternal()
(ImpalaServer.h)
在这个函数里通过 JNI 接口调用 frontend.createExecRequest() 生成 TExecRequest，具体代码如下：

1	Status ImpalaServer::ExecuteInternal(

2	const TQueryCtx& query_ctx,

3	shared_ptr<SessionState> session_state,

4	bool* registered_exec_state,

5	shared_ptr<QueryExecState>* exec_state) {

6	DCHECK(session_state != NULL);

7	*registered_exec_state = false;

8	if (IsOffline()) {

9	return Status("This Impala server is offline. Please retry your query later.");

10

}

11	exec_state->reset(new QueryExecState(query_ctx, exec_env_, exec_env_->frontend(),

12	this, session_state));

13

14	(*exec_state)->query_events()->MarkEvent("Start execution");

15

16	TExecRequest result;

17

{

18	// Keep a lock on exec_state so that registration and setting

19	// result_metadata are atomic.

20

//

21	// Note: this acquires the exec_state lock *before* the

22	// query_exec_state_map_ lock. This is the opposite of

23	// GetQueryExecState(..., true), and therefore looks like a

24	// candidate for deadlock. The reason this works here is that

25	// GetQueryExecState cannot find exec_state (under the exec state

26	// map lock) and take it's lock until RegisterQuery has

27	// finished. By that point, the exec state map lock will have been

28	// given up, so the classic deadlock interleaving is not possible.

29	lock_guard<mutex> l(*(*exec_state)->lock());

30

31	// register exec state as early as possible so that queries that

32	// take a long time to plan show up, and to handle incoming status

33	// reports before execution starts.

34	RETURN_IF_ERROR(RegisterQuery(session_state, *exec_state));

35	*registered_exec_state = true;

36

37	RETURN_IF_ERROR((*exec_state)->UpdateQueryStatus(

38	// 通过 JNI 接口调用 frontend.createExecRequest() 生成 TExecRequest

39	exec_env_->frontend()->GetExecRequest(query_ctx, &result)));

40	(*exec_state)->query_events()->MarkEvent("Planning finished");

41	(*exec_state)->summary_profile()->AddEventSequence(

42	result.timeline.name, result.timeline);

43	if (result.__isset.result_set_metadata) {

44	(*exec_state)->set_result_metadata(result.result_set_metadata);

45

}

46

}

47	VLOG(2) << "Execution request: " << ThriftDebugString(result);

48

49	// start execution of query; also starts fragment status reports

50	RETURN_IF_ERROR((*exec_state)->Exec(&result));

51	if (result.stmt_type == TStmtType::DDL) {

52	Status status = UpdateCatalogMetrics();

53	if (!status.ok()) {

54	VLOG_QUERY << "Couldn't update catalog metrics: " << status.GetDetail();

55

}

56

}

57

58	if ((*exec_state)->coord() != NULL) {

59	const unordered_set<TNetworkAddress>& unique_hosts =

60	(*exec_state)->schedule()->unique_hosts();

61	if (!unique_hosts.empty()) {

62	lock_guard<mutex> l(query_locations_lock_);

63	BOOST_FOREACH(const TNetworkAddress& port, unique_hosts) {

64	query_locations_[port].insert((*exec_state)->query_id());

65

}

66

}

67

}

68	return Status::OK();

69

}

Frontend::GetExecRequest()
(Frontend.h)
通过 JNI 接口调用 frontend.createExecRequest() 生成 TExecRequest。具体实现代码如下：

1	Status Frontend::GetExecRequest(

2	const TQueryCtx& query_ctx, TExecRequest* result) {

3	return JniUtil::CallJniMethod(fe_, create_exec_request_id_, query_ctx, result);

4

}

JniUtil::CallJniMethod()
(jni-util.h)
的具体实现代码如下：

1	/// Utility methods to avoid repeating lots of the JNI call boilerplate. It seems these

2	/// must be defined in the header to compile properly.

3	template <typename T>

4	static Status CallJniMethod(const jobject& obj, const jmethodID& method, const T& arg) {

5	JNIEnv* jni_env = getJNIEnv();

6	jbyteArray request_bytes;

7	JniLocalFrame jni_frame;

8	RETURN_IF_ERROR(jni_frame.push(jni_env));

9	RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &arg, &request_bytes));

10	jni_env->CallObjectMethod(obj, method, request_bytes);

11	RETURN_ERROR_IF_EXC(jni_env);

12	return Status::OK();

13

}

至此，将通过 Thrift 转到 Java Frontend 生成执行计划树。
public TExecRequest createExecRequest(TQueryCtx queryCtx, StringBuilder explainString)
(Frontend.java)
是最重要的方法，它根据提供的 TQueryCtx 创建 TExecRequest。具体代码(分析部分)如下：

1

/**

2	* Create a populated TExecRequest corresponding to the supplied TQueryCtx.

3

*/

4	public TExecRequest createExecRequest(TQueryCtx queryCtx, StringBuilder explainString)

5	throws ImpalaException {

6	// Analyzes the SQL statement included in queryCtx and returns the AnalysisResult.

7	AnalysisContext.AnalysisResult analysisResult = analyzeStmt(queryCtx);

8	EventSequence timeline = analysisResult.getAnalyzer().getTimeline();

9	timeline.markEvent("Analysis finished");

10

.

11

.

12

.

13

.

14

}

首先通过调用 analyzeStmt()
(Frontend.java)
方法分析提交的 SQL 语句。analyzeStmt() 的具体实现代码如下：

1

/**

2	* Analyzes the SQL statement included in queryCtx and returns the AnalysisResult.

3

*/

4	private AnalysisContext.AnalysisResult analyzeStmt(TQueryCtx queryCtx)

5	throws AnalysisException, InternalException, AuthorizationException {

6	AnalysisContext analysisCtx = new AnalysisContext(dsqldCatalog_, queryCtx,

7	authzConfig_);

8	LOG.debug("analyze query " + queryCtx.request.stmt);

9

10	// 循环分析直到出现以下某种情形：

11	// 1) 分析成功完成

12	// 2) 由于缺失表分析失败并抛出 AnalysisException 异常

13	// 3) 分析失败并抛出 AuthorizationException 异常

14

try {

15	while (true) {

16

try {

17	// 通过调用 AnalyzeContex.analyze() 实现具体的分析逻辑

18	analysisCtx.analyze(queryCtx.request.stmt);

19	Preconditions.checkState(analysisCtx.getAnalyzer().getMissingTbls().isEmpty());

20	return analysisCtx.getAnalysisResult();

21	} catch (AnalysisException e) {

22	Set<TableName> missingTbls = analysisCtx.getAnalyzer().getMissingTbls();

23	// Only re-throw the AnalysisException if there were no missing tables.

24	if (missingTbls.isEmpty()) throw e;

25

26	// Some tables/views were missing, request and wait for them to load.

27	if (!requestTblLoadAndWait(missingTbls, MISSING_TBL_LOAD_WAIT_TIMEOUT_MS)) {

28	LOG.info(String.format("Missing tables were not received in %dms. Load " +

29	"request will be retried.", MISSING_TBL_LOAD_WAIT_TIMEOUT_MS));

30

}

31

}

32

}

33	} finally {

34	// Authorize all accesses.

35	// AuthorizationExceptions must take precedence over any AnalysisException

36	// that has been thrown, so perform the authorization first.

37	analysisCtx.getAnalyzer().authorize(getAuthzChecker());

38

}

39

}

AnalyzerContext.AnalyzeResult.Analyzer 对象是个存放这个 SQL 所涉及到的所有信息
（包含Table, conjunct, slot，slotRefMap， eqJoinConjuncts等）的知识库，所有跟这个
SQL 有关的东西都会存到 Analyzer对象里面。该类的定义可以查看
Analyzer.java
AnalyzerContex.analyze()
(AnalyzeContext.java)
的具体实现代码如下：

1

/**

2	* Parse and analyze 'stmt'. If 'stmt' is a nested query (i.e. query that

3	* contains subqueries), it is also rewritten by performing subquery unnesting.

4	* The transformed stmt is then re-analyzed in a new analysis context.

5

*/

6	public void analyze(String stmt) throws AnalysisException {

7	Analyzer analyzer = new Analyzer(catalog_, queryCtx_, authzConfig_);

8	analyze(stmt, analyzer);

9

}

上面的 analyze() 函数通过调用同名的重载函数 analyze(String stmt, Analyzer analyzer)
(AnalyzeContext.java)
实现具体的分析，代码如下:

1

/**

2	* Parse and analyze 'stmt' using a specified Analyzer.

3

*/

4	public void analyze(String stmt, Analyzer analyzer) throws AnalysisException {

5	SqlScanner input = new SqlScanner(new StringReader(stmt));

6	SqlParser parser = new SqlParser(input);

7

try {

8	analysisResult_ = new AnalysisResult();

9	analysisResult_.analyzer_ = analyzer;

10	if (analysisResult_.analyzer_ == null) {

11	analysisResult_.analyzer_ = new Analyzer(catalog_, queryCtx_, authzConfig_);

12

}

13	analysisResult_.stmt_ = (StatementBase) parser.parse().value;

14	if (analysisResult_.stmt_ == null)

15

return;

16

17	// For CTAS(Create Table As Select), we copy the create statement

18	// in case we have to create a new CTAS statement after a query rewrite.

19	if (analysisResult_.stmt_ instanceof CreateTableAsSelectStmt) {

20	analysisResult_.tmpCreateTableStmt_ =

21	((CreateTableAsSelectStmt) analysisResult_.stmt_).getCreateStmt().clone();

22

}

23

24	analysisResult_.stmt_.analyze(analysisResult_.analyzer_);

25	boolean isExplain = analysisResult_.isExplainStmt();

26

27	// Check if we need to rewrite the statement.

28	if (analysisResult_.requiresRewrite()) {

29	StatementBase rewrittenStmt = StmtRewriter.rewrite(analysisResult_);

30	// Re-analyze the rewritten statement.

31	Preconditions.checkNotNull(rewrittenStmt);

32	analysisResult_ = new AnalysisResult();

33	analysisResult_.analyzer_ = new Analyzer(catalog_, queryCtx_, authzConfig_);

34	analysisResult_.stmt_ = rewrittenStmt;

35	analysisResult_.stmt_.analyze(analysisResult_.analyzer_);

36	LOG.trace("rewrittenStmt: " + rewrittenStmt.toSql());

37	if (isExplain)

38	analysisResult_.stmt_.setIsExplain();

39

}

40	} catch (AnalysisException e) {

41	// Don't wrap AnalysisExceptions in another AnalysisException

42

throw e;

43	} catch (Exception e) {

44	throw new AnalysisException(parser.getErrorMsg(stmt), e);

45

}

46

}

上面的函数通过调用 SqlScanner 和 SqlParser 类实现具体的分析。可以查看
sql-scanner.flex
和
sql-parser.y

分析 SQL 语句的大概流程如下：

1. 处理这个 SQL 所涉及到的 Table（即TableRefs），这些 Table 是在 from 从句中提取出来的（包含关键字
   from, join, on/using）。注意 JOIN 操作以及 on/using 条件是存储在参与 JOIN 操作的右边的表的 TableRef
   中并分析的。依次 analyze() 每个 TableRef，向 Analyzer 注册 registerBaseTableRef(填充TupleDescriptor)。
   如果对应的 TableRef 涉及到 JOIN 操作，还要 analyzeJoin()。在 analyzeJoin() 时会向 Analyzer registerConjunct()
   填充 Analyzer 的一些成员变量：conjuncts，tuplePredicates（TupleId 与 conjunct 的映射），slotPredicates（SlotId
   与 conjunct 的映射），eqJoinConjuncts。
2. 处理 select 从句(包含关键字 select, MAX(), AVG()等聚集函数)：分析这个 SQL 都 select 了哪几项，每一项都是个
   Expr 类型的子类对象，把这几项填入 resultExprs 数组和 colLabels。然后把 resultExprs 里面的 Expr 都递归 analyze
   一下，要分析到树的最底层，向 Analyzer 注册 SlotRef 等。
3. 分析 where 从句（关键字 where），首先递归 Analyze 从句中 Expr 组成的树，然后向 Analyzer registerConjunct()
   填充 Analyzer 的一些成员变量(同1，此外还要填充 whereClauseConjuncts) 。
4. 处理 sort 相关信息(关键字 order by)。先是解析 aliases 和 ordinals，然后从 order by 后面的从句中提取 Expr 填入
   orderingExprs，接着递归 Analyze 从句中 Expr 组成的树，最后创建 SortInfo 对象。
5. 处理 aggregation 相关信息（关键字 group by, having, avg, max 等）。首先递归分析 group by 从句里的 Expr，然后如果有
   having 从句就像 where 从句一样，先是 analyze having 从句中 Expr 组成的树，然后向 Analyzer registerConjunct()等。
6. 处理 InlineView。

至此，词法分析和语法分析都完成了，回到 frontend.createExecRequest()
(Frontend.java)
函数，开始填充 TExecRequest 内的成员变量。代码如下(部分)：

1

/**

2	* Create a populated TExecRequest corresponding to the supplied TQueryCtx.

3

*/

4	public TExecRequest createExecRequest(TQueryCtx queryCtx, StringBuilder explainString)

5	throws ImpalaException {

6	// Analyzes the SQL statement included in queryCtx and returns the AnalysisResult.

7	AnalysisContext.AnalysisResult analysisResult = analyzeStmt(queryCtx);

8	EventSequence timeline = analysisResult.getAnalyzer().getTimeline();

9	timeline.markEvent("Analysis finished");

10

11	// 开始填充 TExecRequest

12	Preconditions.checkNotNull(analysisResult.getStmt());

13	TExecRequest result = new TExecRequest();

14	result.setQuery_options(queryCtx.request.getQuery_options());

15	result.setAccess_events(analysisResult.getAccessEvents());

16	result.analysis_warnings = analysisResult.getAnalyzer().getWarnings();

17

18	if (analysisResult.isCatalogOp()) {

19	result.stmt_type = TStmtType.DDL;

20	createCatalogOpRequest(analysisResult, result);

21	String jsonLineageGraph = analysisResult.getJsonLineageGraph();

22	if (jsonLineageGraph != null && !jsonLineageGraph.isEmpty()) {

23	result.catalog_op_request.setLineage_graph(jsonLineageGraph);

24

}

25	// All DDL operations except for CTAS are done with analysis at this point.

26	if (!analysisResult.isCreateTableAsSelectStmt()) return result;

27	} else if (analysisResult.isLoadDataStmt()) {

28	result.stmt_type = TStmtType.LOAD;

29	result.setResult_set_metadata(new TResultSetMetadata(Arrays.asList(

30	new TColumn("summary", Type.STRING.toThrift()))));

31	result.setLoad_data_request(analysisResult.getLoadDataStmt().toThrift());

32	return result;

33	} else if (analysisResult.isSetStmt()) {

34	result.stmt_type = TStmtType.SET;

35	result.setResult_set_metadata(new TResultSetMetadata(Arrays.asList(

36	new TColumn("option", Type.STRING.toThrift()),

37	new TColumn("value", Type.STRING.toThrift()))));

38	result.setSet_query_option_request(analysisResult.getSetStmt().toThrift());

39	return result;

40

}

41

.

42

.

43

.

44

.

45

46

}

如果是 DDL 命令（use, show tables, show databases, describe），那么调用 createCatalogOpRequest()。
如果是 Load Data 或者 Set 语句，就调用相应的 setmetadata 并转换为 Thrift。

执行计划生成

另外一种情况就是 Query 或者 DML 命令，那么就得创建和填充 TQueryExecRequest 了。该部分代码如下：

1

/**

2	* Create a populated TExecRequest corresponding to the supplied TQueryCtx.

3

*/

4	public TExecRequest createExecRequest(TQueryCtx queryCtx, StringBuilder explainString)

5	throws DsqlException {

6

.

7

.

8

.

9

.

10

.

11	// create TQueryExecRequest 如果是 Query、DML、或 CTAS 语句

12	Preconditions.checkState(analysisResult.isQueryStmt() || analysisResult.isDmlStmt()

13	|| analysisResult.isCreateTableAsSelectStmt());

14

15	TQueryExecRequest queryExecRequest = new TQueryExecRequest();

16	// create plan

17	LOG.debug("create plan");

18	Planner planner = new Planner(analysisResult, queryCtx);

19	// 根据 SQL 语法树生成执行计划(PlanNode 和 PlanFragment)

20	// 用 Planner 把 SQL 解析出的语法树转换成 Plan fragments，后者能在各个 backend 被执行。

21	ArrayList<PlanFragment> fragments = planner.createPlan();

22

23	List<ScanNode> scanNodes = Lists.newArrayList();

24	// 建立 queryExecRequest.fragments 中 fragment 到它索引的映射;

25	// queryExecRequest.dest_fragment_idx 需要这些映射

26	Map<PlanFragment, Integer> fragmentIdx = Maps.newHashMap();

27

28	for (int fragmentId = 0; fragmentId < fragments.size(); ++fragmentId) {

29	PlanFragment fragment = fragments.get(fragmentId);

30	Preconditions.checkNotNull(fragment.getPlanRoot());

31	fragment.getPlanRoot().collect(Predicates.instanceOf(ScanNode.class), scanNodes);

32	fragmentIdx.put(fragment, fragmentId);

33

}

34

.

35

.

36

.

37

.

38

.

39

}

上面的 createPlan() 函数是 frontend 最重要的函数：根据 SQL 解析的结果和 client 传入的 query options，
生成执行计划。执行计划是用 PlanFragment 的数组表示的，最后会序列化到 TQueryExecRequest.fragments
然后传给 backend 的 coordinator 去调度执行。现在让我们来看看 createPlan()
(Planner.java)
的具体实现：

1

/**

2	* Returns a list of plan fragments for executing an analyzed parse tree.

3	* May return a single-node or distributed executable plan.

4

*/

5	public ArrayList<PlanFragment> createPlan() throws ImpalaException {

6	SingleNodePlanner singleNodePlanner = new SingleNodePlanner(ctx_);

7	DistributedPlanner distributedPlanner = new DistributedPlanner(ctx_);

8	// 首先生成 SingleNodePlan，单节点执行计划树

9	PlanNode singleNodePlan = singleNodePlanner.createSingleNodePlan();

10	ctx_.getRootAnalyzer().getTimeline().markEvent("Single node plan created");

11	ArrayList<PlanFragment> fragments = null;

12

13	// Determine the maximum number of rows processed by any node in the plan tree

14	MaxRowsProcessedVisitor visitor = new MaxRowsProcessedVisitor();

15	singleNodePlan.accept(visitor);

16	long maxRowsProcessed = visitor.get() == -1 ? Long.MAX_VALUE : visitor.get();

17	boolean isSmallQuery =

18	maxRowsProcessed < ctx_.getQueryOptions().exec_single_node_rows_threshold;

19	if (isSmallQuery) {

20	// Execute on a single node and disable codegen for small results

21	ctx_.getQueryOptions().setNum_nodes(1);

22	ctx_.getQueryOptions().setDisable_codegen(true);

23	if (maxRowsProcessed < ctx_.getQueryOptions().batch_size ||

24	maxRowsProcessed < 1024 && ctx_.getQueryOptions().batch_size == 0) {

25	// Only one scanner thread for small queries

26	ctx_.getQueryOptions().setNum_scanner_threads(1);

27

}

28

}

29

30	if (ctx_.isSingleNodeExec()) {// 如果是单节点执行计划树

31	// 创建保护整个单点计划树的片段

32	fragments = Lists.newArrayList(new PlanFragment(

33	ctx_.getNextFragmentId(), singleNodePlan, DataPartition.UNPARTITIONED));

34	} else {// 分布式执行计划树

35	// create distributed plan

36	fragments = distributedPlanner.createPlanFragments(singleNodePlan);

37

}

38	// 最后一个 Fragment 是根 fragment

39	PlanFragment rootFragment = fragments.get(fragments.size() - 1);

40	if (ctx_.isInsertOrCtas()) {

41	InsertStmt insertStmt = ctx_.getAnalysisResult().getInsertStmt();

42	if (!ctx_.isSingleNodeExec()) {

43	// repartition on partition keys

44	rootFragment = distributedPlanner.createInsertFragment(

45	rootFragment, insertStmt, ctx_.getRootAnalyzer(), fragments);

46

}

47	// set up table sink for root fragment

48	rootFragment.setSink(insertStmt.createDataSink());

49

}

50

51	ColumnLineageGraph graph = ctx_.getRootAnalyzer().getColumnLineageGraph();

52	List<Expr> resultExprs = null;

53	Table targetTable = null;

54	if (ctx_.isInsertOrCtas()) {

55	InsertStmt insertStmt = ctx_.getAnalysisResult().getInsertStmt();

56	resultExprs = insertStmt.getResultExprs();

57	targetTable = insertStmt.getTargetTable();

58	graph.addTargetColumnLabels(targetTable);

59

} else {

60	resultExprs = ctx_.getQueryStmt().getResultExprs();

61	graph.addTargetColumnLabels(ctx_.getQueryStmt().getColLabels());

62

}

63	resultExprs = Expr.substituteList(resultExprs,

64	rootFragment.getPlanRoot().getOutputSmap(), ctx_.getRootAnalyzer(), true);

65	rootFragment.setOutputExprs(resultExprs);

66	LOG.debug("desctbl: " + ctx_.getRootAnalyzer().getDescTbl().debugString());

67	LOG.debug("resultexprs: " + Expr.debugString(rootFragment.getOutputExprs()));

68	LOG.debug("finalize plan fragments");

69	for (PlanFragment fragment: fragments) {

70	fragment.finalize(ctx_.getRootAnalyzer());

71

}

72

73	Collections.reverse(fragments);

74	ctx_.getRootAnalyzer().getTimeline().markEvent("Distributed plan created");

75

76	if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {

77	// Compute the column lineage graph

78	if (ctx_.isInsertOrCtas()) {

79	Preconditions.checkNotNull(targetTable);

80	List<Expr> exprs = Lists.newArrayList();

81	if (targetTable instanceof HBaseTable) {

82	exprs.addAll(resultExprs);

83

} else {

84	exprs.addAll(ctx_.getAnalysisResult().getInsertStmt().getPartitionKeyExprs());

85	exprs.addAll(resultExprs.subList(0,

86	targetTable.getNonClusteringColumns().size()));

87

}

88	graph.computeLineageGraph(exprs, ctx_.getRootAnalyzer());

89

} else {

90	graph.computeLineageGraph(resultExprs, ctx_.getRootAnalyzer());

91

}

92	LOG.trace("lineage: " + graph.debugString());

93	ctx_.getRootAnalyzer().getTimeline().markEvent("Lineage info computed");

94

}

95

96	return fragments;

97

}

createPlan 包括createSingleNodePlan 和 createPlanFragments
两个主要部分。其中第一个是单节点计划树，所有片段只能在一个节点 corrd 上执行，第二个是分布式执行计划树，片段可以分配到不同的节点中运行。我们先来看看 SingleNodePlanner.createSingleNodePlan()
(SingleNodePlanner.java)
该方法根据 Planner Context 中分析的语法树创建单节点执行计划树并返回根节点。计划递归处理语法树并执行以下操作，自上而下处理查询语句：

1. materialize the slots required for evaluating expressions of that statement
2. migrate conjuncts from parent blocks into inline views and union operands In the bottom-up phase generate the plan tree for every query statement:
3. perform join-order optimization when generating the plan of the FROM clause of a select statement; requires that all materialized slots are known for an accurate estimate of row sizes needed for cost-based join ordering
4. assign conjuncts that can be evaluated at that node and compute the stats of that node (cardinality, etc.)
5. apply combined expression substitution map of child plan nodes; if a plan node re-maps its input, set a substitution map to be applied by parents

具体代码如下：

1

/**

2	* Generates and returns the root of the single-node plan for the analyzed parse tree

3	* in the planner context.

4

*/

5	public PlanNode createSingleNodePlan() throws ImpalaException {

6	QueryStmt queryStmt = ctx_.getQueryStmt();

7	// Use the stmt's analyzer which is not necessarily the root analyzer

8	// to detect empty result sets.

9	Analyzer analyzer = queryStmt.getAnalyzer();

10	analyzer.computeEquivClasses();

11	analyzer.getTimeline().markEvent("Equivalence classes computed");

12

13	// Mark slots referenced by output exprs as materialized, prior to generating the

14	// plan tree.

15	// We need to mark the result exprs of the topmost select block as materialized, so

16	// that PlanNode.init() can compute the final mem layout of materialized tuples

17	// (the byte size of tuples is needed for cost computations).

18	// TODO: instead of materializing everything produced by the plan root, derive

19	// referenced slots from destination fragment and add a materialization node

20	// if not all output is needed by destination fragment

21	// TODO 2: should the materialization decision be cost-based?

22	if (queryStmt.getBaseTblResultExprs() != null) {

23	analyzer.materializeSlots(queryStmt.getBaseTblResultExprs());

24

}

25

26	LOG.trace("desctbl: " + analyzer.getDescTbl().debugString());

27	PlanNode singleNodePlan = createQueryPlan(queryStmt, analyzer,

28	ctx_.getQueryOptions().isDisable_outermost_topn());

29	Preconditions.checkNotNull(singleNodePlan);

30	return singleNodePlan;

31

}

上面的函数通过调用私有的 createQueryPlan()
(SingleNodePlanner.java)
函数实现。该函数为单节点执行创建计划树。为查询语句中的
Select/Project/Join/Union [All]/Group by/Having/Order by
生成 PlanNode。具体实现代码如下：

1

/**

2	* Create plan tree for single-node execution. Generates PlanNodes for the

3	* Select/Project/Join/Union [All]/Group by/Having/Order by clauses of the query stmt.

4

*/

5	private PlanNode createQueryPlan(QueryStmt stmt, Analyzer analyzer, booleandisableTopN)

6	throws ImpalaException {

7	// Analyzer 检测结果集是否为空，如果是的话直接返回空节点

8	if (analyzer.hasEmptyResultSet()) return createEmptyNode(stmt, analyzer);

9

10	PlanNode root;

11	if (stmt instanceof SelectStmt) {// 如果是 select 语句

12	SelectStmt selectStmt = (SelectStmt) stmt;

13	// 创建 SelectPlan

14	root = createSelectPlan(selectStmt, analyzer);

15

16	// insert possible AnalyticEvalNode before SortNode

17	if (((SelectStmt) stmt).getAnalyticInfo() != null) {

18	AnalyticInfo analyticInfo = selectStmt.getAnalyticInfo();

19	ArrayList<TupleId> stmtTupleIds = Lists.newArrayList();

20	stmt.getMaterializedTupleIds(stmtTupleIds);

21	AnalyticPlanner analyticPlanner =

22	new AnalyticPlanner(stmtTupleIds, analyticInfo, analyzer, ctx_);

23	List<Expr> inputPartitionExprs = Lists.newArrayList();

24	AggregateInfo aggInfo = selectStmt.getAggInfo();

25	root = analyticPlanner.createSingleNodePlan(root,

26	aggInfo != null ? aggInfo.getGroupingExprs() : null, inputPartitionExprs);

27	if (aggInfo != null && !inputPartitionExprs.isEmpty()) {

28	// analytic computation will benefit from a partition on inputPartitionExprs

29	aggInfo.setPartitionExprs(inputPartitionExprs);

30

}

31

}

32	} else {// 否则，创建 UnionPlan

33	Preconditions.checkState(stmt instanceof UnionStmt);

34	root = createUnionPlan((UnionStmt) stmt, analyzer);

35

}

36

37	// 如果 sort 元组有没有物化的槽，避免添加 sort node，

38	boolean sortHasMaterializedSlots = false;

39	if (stmt.evaluateOrderBy()) {

40	for (SlotDescriptor sortSlotDesc:

41	stmt.getSortInfo().getSortTupleDescriptor().getSlots()) {

42	if (sortSlotDesc.isMaterialized()) {

43	sortHasMaterializedSlots = true;

44

break;

45

}

46

}

47

}

48

49	if (stmt.evaluateOrderBy() && sortHasMaterializedSlots) {

50	long limit = stmt.getLimit();

51	// TODO: External sort could be used for very large limits

52	// not just unlimited order-by

53	boolean useTopN = stmt.hasLimit() && !disableTopN;

54	// 创建 sort node

55	root = new SortNode(ctx_.getNextNodeId(), root, stmt.getSortInfo(),

56	useTopN, stmt.getOffset());

57	Preconditions.checkState(root.hasValidStats());

58	root.setLimit(limit);

59	root.init(analyzer);

60

} else {

61	root.setLimit(stmt.getLimit());

62	root.computeStats(analyzer);

63

}

64

65	return root;

66

}

SingleNodePlanner.createSelectPlan()
(SingleNodePlanner.java)
函数创建实现 select 查询语句块中
Select/Project/Join/Group by/Having 等从句的 PlanNode 树。具体实现代码如下：

1

/**

2	* Create tree of PlanNodes that implements the Select/Project/Join/Group by/Having

3	* of the selectStmt query block.

4

*/

5	private PlanNode createSelectPlan(SelectStmt selectStmt, Analyzer analyzer)

6	throws ImpalaException {

7	// no from clause -> materialize the select's exprs with a UnionNode

8	// 如果 select 语句没有引用任何 table，创建 ConstantSelectPlan

9	if (selectStmt.getTableRefs().isEmpty()) {

10	return createConstantSelectPlan(selectStmt, analyzer);

11

}

12

13	// Slot materialization:

14	// We need to mark all slots as materialized that are needed during the execution

15	// of selectStmt, and we need to do that prior to creating plans for the TableRefs

16	// (because createTableRefNode() might end up calling computeMemLayout() on one or

17	// more TupleDescriptors, at which point all referenced slots need to be marked).

18

//

19	// For non-join predicates, slots are marked as follows:

20	// - for base table scan predicates, this is done directly by ScanNode.init(), which

21	//   can do a better job because it doesn't need to materialize slots that are only

22	//   referenced for partition pruning, for instance

23	// - for inline views, non-join predicates are pushed down, at which point the

24	//   process repeats itself.

25	selectStmt.materializeRequiredSlots(analyzer);

26

27	ArrayList<TupleId> rowTuples = Lists.newArrayList();

28	// collect output tuples of subtrees

29	for (TableRef tblRef: selectStmt.getTableRefs()) {

30	rowTuples.addAll(tblRef.getMaterializedTupleIds());

31

}

32

33	// 如果 select 语句中的 select、project、join 部分返回空结果集

34	// 用空集创建满足 select 语句的 AggregationPlan

35	// Make sure the slots of the aggregation exprs and the tuples that they reference

36	// are materialized (see IMPALA-1960).

37	if (analyzer.hasEmptySpjResultSet()) {

38	PlanNode emptySetNode = new EmptySetNode(ctx_.getNextNodeId(), rowTuples);

39	emptySetNode.init(analyzer);

40	emptySetNode.setOutputSmap(selectStmt.getBaseTblSmap());

41	return createAggregationPlan(selectStmt, analyzer, emptySetNode);

42

}

43

44	// 为 table refs 创建 Plan；这里使用 list 而不是 map 是为了保证生成 join plan

45	// 时遍历 TableRefs 有一个确定的顺序

46	List<Pair<TableRef, PlanNode>> refPlans = Lists.newArrayList();

47	for (TableRef ref: selectStmt.getTableRefs()) {

48	PlanNode plan = createTableRefNode(analyzer, ref);

49	Preconditions.checkState(plan != null);

50	refPlans.add(new Pair(ref, plan));

51

}

52	// save state of conjunct assignment; needed for join plan generation

53	for (Pair<TableRef, PlanNode> entry: refPlans) {

54	entry.second.setAssignedConjuncts(analyzer.getAssignedConjuncts());

55

}

56

57	PlanNode root = null;

58	// 如果有足够的统计数据，例如 join 操作各个 table 的大小，创建开销最小的 JoinPlan

59	if (!selectStmt.getSelectList().isStraightJoin()) {

60	Set<ExprId> assignedConjuncts = analyzer.getAssignedConjuncts();

61	root = createCheapestJoinPlan(analyzer, refPlans);

62	if (root == null) analyzer.setAssignedConjuncts(assignedConjuncts);

63

}

64	// 否则，根据 from 从句中 table 顺序创建 JoinPlan

65	if (selectStmt.getSelectList().isStraightJoin() || root == null) {

66	// we didn't have enough stats to do a cost-based join plan, or the STRAIGHT_JOIN

67	// keyword was in the select list: use the FROM clause order instead

68	root = createFromClauseJoinPlan(analyzer, refPlans);

69	Preconditions.checkNotNull(root);

70

}

71

72	// 如果有聚集操作，创建 AggregationPlan

73	if (selectStmt.getAggInfo() != null) {

74	root = createAggregationPlan(selectStmt, analyzer, root);

75

}

76

77	// All the conjuncts_ should be assigned at this point.

78	// TODO: Re-enable this check here and/or elswehere.

79	//Preconditions.checkState(!analyzer.hasUnassignedConjuncts());

80	return root;

81

}

上面函数中调用的主要私有方法有：
createTableRefNode()、createCheapestJoinPlan()、 createFromClauseJoinPlan()、 createAggregationPlan()，各个函数的具体实现如下：

createTableRefNode()

1

/**

2	* Create a tree of PlanNodes for the given tblRef, which can be a BaseTableRef,

3	* CollectionTableRef or an InlineViewRef.

4

*/

5	private PlanNode createTableRefNode(Analyzer analyzer, TableRef tblRef)

6	throws ImpalaException {

7	if (tblRef instanceof BaseTableRef || tblRef instanceof CollectionTableRef) {

8	// 创建 ScanNode

9	return createScanNode(analyzer, tblRef);

10	} else if (tblRef instanceof InlineViewRef) {

11	// 创建 InlineViewPlan

12	return createInlineViewPlan(analyzer, (InlineViewRef) tblRef);

13

}

14	throw new InternalException(

15	"Unknown TableRef node: " + tblRef.getClass().getSimpleName());

16

}

createCheapestJoinPlan()

1

/**

2	* 返回物化 join refPlans 中所有 TblRefs 开销最小的 plan

3	* 假设 refPlans 中的顺序和查询中的原始顺序相同

4	* For this plan:

5	* - the plan is executable, ie, all non-cross joins have equi-join predicates

6	* - the leftmost scan is over the largest of the inputs for which we can still

7	*   construct an executable plan(左边的是最大表)

8	* - all rhs's(right hand side?) are in decreasing order of selectiveness (percentage of rows they

9	*   eliminate)

10	* - outer/cross/semi joins: rhs serialized size is < lhs serialized size;(右边的表比左边的小)

11	*   enforced via join inversion, if necessary(否则通过 join 反转实现)

12	* Returns null if we can't create an executable plan.

13

*/

14	private PlanNode createCheapestJoinPlan(

15	Analyzer analyzer, List<Pair<TableRef, PlanNode>> refPlans)

16	throws ImpalaException {

17	LOG.trace("createCheapestJoinPlan");

18	if (refPlans.size() == 1) return refPlans.get(0).second;

19

20	// collect eligible candidates for the leftmost input; list contains

21	// (plan, materialized size)

22	ArrayList<Pair<TableRef, Long>> candidates = Lists.newArrayList();

23	for (Pair<TableRef, PlanNode> entry: refPlans) {

24	TableRef ref = entry.first;

25	JoinOperator joinOp = ref.getJoinOp();

26

27	// The rhs table of an outer/semi join can appear as the left-most input if we

28	// invert the lhs/rhs and the join op. However, we may only consider this inversion

29	// for the very first join in refPlans, otherwise we could reorder tables/joins

30	// across outer/semi joins which is generally incorrect. The null-aware

31	// left anti-join operator is never considered for inversion because we can't

32	// execute the null-aware right anti-join efficiently.

33	// TODO: Allow the rhs of any cross join as the leftmost table. This needs careful

34	// consideration of the joinOps that result from such a re-ordering (IMPALA-1281).

35	if (((joinOp.isOuterJoin() || joinOp.isSemiJoin() || joinOp.isCrossJoin()) &&

36	ref != refPlans.get(1).first) || joinOp.isNullAwareLeftAntiJoin()) {

37	// ref cannot appear as the leftmost input

38

continue;

39

}

40

41	PlanNode plan = entry.second;

42	if (plan.getCardinality() == -1) {

43	// use 0 for the size to avoid it becoming the leftmost input

44	// TODO: Consider raw size of scanned partitions in the absence of stats.

45	candidates.add(new Pair(ref, new Long(0)));

46	LOG.trace("candidate " + ref.getUniqueAlias() + ": 0");

47

continue;

48

}

49	Preconditions.checkNotNull(ref.getDesc());

50	long materializedSize =

51	(long) Math.ceil(plan.getAvgRowSize() * (double) plan.getCardinality());

52	candidates.add(new Pair(ref, new Long(materializedSize)));

53	LOG.trace("candidate " + ref.getUniqueAlias() + ": " + Long.toString(materializedSize));

54

}

55	if (candidates.isEmpty()) return null;

56

57	// order candidates by descending materialized size; we want to minimize the memory

58	// consumption of the materialized hash tables required for the join sequence

59	Collections.sort(candidates,

60	new Comparator<Pair<TableRef, Long>>() {

61	public int compare(Pair<TableRef, Long> a, Pair<TableRef, Long> b) {

62	long diff = b.second - a.second;

63	return (diff < 0 ? -1 : (diff > 0 ? 1 : 0));

64

}

65

});

66

67	// 根据已经按照大小排序的 table 创建 JoinPlan

68	for (Pair<TableRef, Long> candidate: candidates) {

69	PlanNode result = createJoinPlan(analyzer, candidate.first, refPlans);

70	if (result != null) return result;

71

}

72	return null;

73

}

createFromClauseJoinPlan()

1

/**

2	* 返回按照 from 语句顺序的 JoinPlan

3

*/

4	private PlanNode createFromClauseJoinPlan(

5	Analyzer analyzer, List<Pair<TableRef, PlanNode>> refPlans)

6	throws ImpalaException {

7	// create left-deep sequence of binary hash joins; assign node ids as we go along

8	Preconditions.checkState(!refPlans.isEmpty());

9	PlanNode root = refPlans.get(0).second;

10	for (int i = 1; i < refPlans.size(); ++i) {

11	TableRef innerRef = refPlans.get(i).first;

12	PlanNode innerPlan = refPlans.get(i).second;

13	root = createJoinNode(analyzer, root, innerPlan, null, innerRef);

14	root.setId(ctx_.getNextNodeId());

15

}

16	return root;

17

}

createAggregationPlan()

1

/**

2	* Returns a new AggregationNode that materializes the aggregation of the given stmt.

3	* Assigns conjuncts from the Having clause to the returned node.

4

*/

5	private PlanNode createAggregationPlan(SelectStmt selectStmt, Analyzer analyzer,

6	PlanNode root) throws InternalException {

7	Preconditions.checkState(selectStmt.getAggInfo() != null);

8	// add aggregation, if required

9	AggregateInfo aggInfo = selectStmt.getAggInfo();

10	root = new AggregationNode(ctx_.getNextNodeId(), root, aggInfo);

11	root.init(analyzer);

12	Preconditions.checkState(root.hasValidStats());

13	// if we're computing DISTINCT agg fns, the analyzer already created the

14	// 2nd phase agginfo

15	if (aggInfo.isDistinctAgg()) {

16	((AggregationNode)root).unsetNeedsFinalize();

17	// The output of the 1st phase agg is the 1st phase intermediate.

18	((AggregationNode)root).setIntermediateTuple();

19	root = new AggregationNode(ctx_.getNextNodeId(), root,

20	aggInfo.getSecondPhaseDistinctAggInfo());

21	root.init(analyzer);

22	Preconditions.checkState(root.hasValidStats());

23

}

24	// add Having clause

25	root.assignConjuncts(analyzer);

26	return root;

27

}

上面的 createCheapestJoinPlan() 和 createFromClauseJoinPlan()
方法调用了 createJoinNode() 和 createJoinPlan() 两个方法。它们的具体实现如下：

createJoinNode()

1

/**

2	* 创建 join outer 和 inner 的 node。两者其中之一可能是一个根据 table ref 创建的 plan

3	* 但不能同时都是 plan。对应的 outer/inner tableRef 不能为空

4

*/

5	private PlanNode createJoinNode(

6	Analyzer analyzer, PlanNode outer, PlanNode inner, TableRef outerRef,

7	TableRef innerRef) throws ImpalaException {

8	Preconditions.checkState(innerRef != null ^ outerRef != null);

9	TableRef tblRef = (innerRef != null) ? innerRef : outerRef;

10

11	List<BinaryPredicate> eqJoinConjuncts = Lists.newArrayList();

12	List<Expr> eqJoinPredicates = Lists.newArrayList();

13	// get eq join predicates for the TableRefs' ids (not the PlanNodes' ids, which

14	// are materialized)

15	if (innerRef != null) {

16	getHashLookupJoinConjuncts(

17	analyzer, outer.getTblRefIds(), innerRef, eqJoinConjuncts, eqJoinPredicates);

18	// Outer joins should only use On-clause predicates as eqJoinConjuncts.

19	if (!innerRef.getJoinOp().isOuterJoin()) {

20	analyzer.createEquivConjuncts(outer.getTblRefIds(), innerRef.getId(),

21	eqJoinConjuncts);

22

}

23

} else {

24	getHashLookupJoinConjuncts(

25	analyzer, inner.getTblRefIds(), outerRef, eqJoinConjuncts, eqJoinPredicates);

26	// Outer joins should only use On-clause predicates as eqJoinConjuncts.

27	if (!outerRef.getJoinOp().isOuterJoin()) {

28	analyzer.createEquivConjuncts(inner.getTblRefIds(), outerRef.getId(),

29	eqJoinConjuncts);

30

}

31	// Reverse the lhs/rhs of the join conjuncts.

32	for (BinaryPredicate eqJoinConjunct: eqJoinConjuncts) {

33	Expr swapTmp = eqJoinConjunct.getChild(0);

34	eqJoinConjunct.setChild(0, eqJoinConjunct.getChild(1));

35	eqJoinConjunct.setChild(1, swapTmp);

36

}

37

}

38

39	// 处理隐含交叉 join

40	if (eqJoinConjuncts.isEmpty()) {

41	// Since our only implementation of semi and outer joins is hash-based, and we do

42	// not re-order semi and outer joins, we must have eqJoinConjuncts here to execute

43	// this query.

44	// TODO: Revisit when we add more semi/join implementations. Pick up and pass in

45	// the otherJoinConjuncts.

46	if (tblRef.getJoinOp().isOuterJoin() ||

47	tblRef.getJoinOp().isSemiJoin()) {

48	throw new NotImplementedException(

49	String.format("%s join with '%s' without equi-join " +

50	"conjuncts is not supported.",

51	tblRef.getJoinOp().isOuterJoin() ? "Outer" : "Semi",

52	innerRef.getUniqueAlias()));

53

}

54	CrossJoinNode result =

55	new CrossJoinNode(outer, inner, tblRef, Collections.<Expr>emptyList());

56	result.init(analyzer);

57	return result;

58

}

59

60	// 处理显式交叉 join

61	if (tblRef.getJoinOp() == JoinOperator.CROSS_JOIN) {

62	tblRef.setJoinOp(JoinOperator.INNER_JOIN);

63

}

64

65	analyzer.markConjunctsAssigned(eqJoinPredicates);

66

67	List<Expr> otherJoinConjuncts = Lists.newArrayList();

68	if (tblRef.getJoinOp().isOuterJoin()) {// 外连接

69	// Also assign conjuncts from On clause. All remaining unassigned conjuncts

70	// that can be evaluated by this join are assigned in createSelectPlan().

71	otherJoinConjuncts = analyzer.getUnassignedOjConjuncts(tblRef);

72	} else if (tblRef.getJoinOp().isSemiJoin()) {// 半连接

73	// Unassigned conjuncts bound by the invisible tuple id of a semi join must have

74	// come from the join's On-clause, and therefore, must be added to the other join

75	// conjuncts to produce correct results.

76	otherJoinConjuncts =

77	analyzer.getUnassignedConjuncts(tblRef.getAllTupleIds(), false);

78	if (tblRef.getJoinOp().isNullAwareLeftAntiJoin()) {// 对空值敏感的反连接

79	boolean hasNullMatchingEqOperator = false;

80	// Keep only the null-matching eq conjunct in the eqJoinConjuncts and move

81	// all the others in otherJoinConjuncts. The BE relies on this

82	// separation for correct execution of the null-aware left anti join.

83	Iterator<BinaryPredicate> it = eqJoinConjuncts.iterator();

84	while (it.hasNext()) {

85	BinaryPredicate conjunct = it.next();

86	if (!conjunct.isNullMatchingEq()) {

87	otherJoinConjuncts.add(conjunct);

88	it.remove();

89

} else {

90	// Only one null-matching eq conjunct is allowed

91	Preconditions.checkState(!hasNullMatchingEqOperator);

92	hasNullMatchingEqOperator = true;

93

}

94

}

95	Preconditions.checkState(hasNullMatchingEqOperator);

96

}

97

}

98	analyzer.markConjunctsAssigned(otherJoinConjuncts);

99

100	HashJoinNode result =

101	new HashJoinNode(outer, inner, tblRef, eqJoinConjuncts, otherJoinConjuncts);

102	result.init(analyzer);

103	return result;

104

}

createJoinPlan()

1

/**

2	* Returns a plan with leftmostRef's plan as its leftmost input; the joins

3	* are in decreasing order of selectiveness (percentage of rows they eliminate).

4	* The leftmostRef's join will be inverted if it is an outer/semi/cross join.

5

*/

6	private PlanNode createJoinPlan(

7	Analyzer analyzer, TableRef leftmostRef, List<Pair<TableRef, PlanNode>> refPlans)

8	throws ImpalaException {

9

10	LOG.trace("createJoinPlan: " + leftmostRef.getUniqueAlias());

11	// 等待 join 的 tableref

12	List<Pair<TableRef, PlanNode>> remainingRefs = Lists.newArrayList();

13	PlanNode root = null;  // root of accumulated join plan

14	for (Pair<TableRef, PlanNode> entry: refPlans) {

15	if (entry.first == leftmostRef) {

16	root = entry.second;

17

} else {

18	remainingRefs.add(entry);

19

}

20

}

21	Preconditions.checkNotNull(root);

22	// 已经 join 的 refs；joinedRefs 和 remainingRefs 中 refs 的 union 就是所有 table refs

23	Set<TableRef> joinedRefs = Sets.newHashSet();

24	joinedRefs.add(leftmostRef);

25

26	// 如果最左边的 TblRef 是 outer/semi/cross join，反转

27	boolean planHasInvertedJoin = false;

28	if (leftmostRef.getJoinOp().isOuterJoin()

29	|| leftmostRef.getJoinOp().isSemiJoin()

30	|| leftmostRef.getJoinOp().isCrossJoin()) {

31	// TODO: Revisit the interaction of join inversion here and the analysis state

32	// that is changed in analyzer.invertOuterJoin(). Changing the analysis state

33	// should not be necessary because the semantics of an inverted outer join do

34	// not change.

35	leftmostRef.invertJoin(refPlans, analyzer);

36	planHasInvertedJoin = true;

37

}

38

39	long numOps = 0;

40	int i = 0;

41	while (!remainingRefs.isEmpty()) {

42	// Join 链中的每一步都最小化结果数目，从而最小化 hash table 查找

43	PlanNode newRoot = null;

44	Pair<TableRef, PlanNode> minEntry = null;

45	for (Pair<TableRef, PlanNode> entry: remainingRefs) {

46	TableRef ref = entry.first;

47	LOG.trace(Integer.toString(i) + " considering ref " + ref.getUniqueAlias());

48

49	// Determine whether we can or must consider this join at this point in the plan.

50	// Place outer/semi joins at a fixed position in the plan tree (IMPALA-860),

51	// s.t. all the tables appearing to the left/right of an outer/semi join in

52	// the original query still remain to the left/right after join ordering. This

53	// prevents join re-ordering across outer/semi joins which is generally wrong.

54	// The checks below relies on remainingRefs being in the order as they originally

55	// appeared in the query.

56	JoinOperator joinOp = ref.getJoinOp();

57	if (joinOp.isOuterJoin() || joinOp.isSemiJoin()) {

58	List<TupleId> currentTids = Lists.newArrayList(root.getTblRefIds());

59	currentTids.add(ref.getId());

60	// Place outer/semi joins at a fixed position in the plan tree. We know that

61	// the join resulting from 'ref' must become the new root if the current

62	// root materializes exactly those tuple ids corresponding to TableRefs

63	// appearing to the left of 'ref' in the original query.

64	List<TupleId> tableRefTupleIds = ref.getAllTupleIds();

65	if (!currentTids.containsAll(tableRefTupleIds) ||

66	!tableRefTupleIds.containsAll(currentTids)) {

67	// Do not consider the remaining table refs to prevent incorrect re-ordering

68	// of tables across outer/semi/anti joins.

69

break;

70

}

71	} else if (ref.getJoinOp().isCrossJoin()) {

72	if (!joinedRefs.contains(ref.getLeftTblRef())) continue;

73

}

74

75	PlanNode rhsPlan = entry.second;

76	analyzer.setAssignedConjuncts(root.getAssignedConjuncts());

77

78	boolean invertJoin = false;

79	if (joinOp.isOuterJoin() || joinOp.isSemiJoin() || joinOp.isCrossJoin()) {

80	// Invert the join if doing so reduces the size of build-side hash table

81	// (may also reduce network costs depending on the join strategy).

82	// Only consider this optimization if both the lhs/rhs cardinalities are known.

83	// The null-aware left anti-join operator is never considered for inversion

84	// because we can't execute the null-aware right anti-join efficiently.

85	long lhsCard = root.getCardinality();

86	long rhsCard = rhsPlan.getCardinality();

87	if (lhsCard != -1 && rhsCard != -1 &&

88	lhsCard * root.getAvgRowSize() < rhsCard * rhsPlan.getAvgRowSize() &&

89	!joinOp.isNullAwareLeftAntiJoin()) {

90	invertJoin = true;

91

}

92

}

93	PlanNode candidate = null;

94	if (invertJoin) {

95	ref.setJoinOp(ref.getJoinOp().invert());

96	candidate = createJoinNode(analyzer, rhsPlan, root, ref, null);

97	planHasInvertedJoin = true;

98

} else {

99	candidate = createJoinNode(analyzer, root, rhsPlan, null, ref);

100

}

101	if (candidate == null) continue;

102	LOG.trace("cardinality=" + Long.toString(candidate.getCardinality()));

103

104	// Use 'candidate' as the new root; don't consider any other table refs at this

105	// position in the plan.

106	if (joinOp.isOuterJoin() || joinOp.isSemiJoin()) {

107	newRoot = candidate;

108	minEntry = entry;

109

break;

110

}

111

112	// 优先选择 Hash Join 而不是 Cross Join， due to limited costing infrastructure

113	if (newRoot == null

114	|| (candidate.getClass().equals(newRoot.getClass())

115	&& candidate.getCardinality() < newRoot.getCardinality())

116	|| (candidate instanceof HashJoinNode && newRoot instanceofCrossJoinNode)) {

117	newRoot = candidate;

118	minEntry = entry;

119

}

120

}

121	if (newRoot == null) {

122	// Currently, it should not be possible to invert a join for a plan that turns

123	// out to be non-executable because (1) the joins we consider for inversion are

124	// barriers in the join order, and (2) the caller of this function only considers

125	// other leftmost table refs if a plan turns out to be non-executable.

126	// TODO: This preconditions check will need to be changed to undo the in-place

127	// modifications made to table refs for join inversion, if the caller decides to

128	// explore more leftmost table refs.

129	Preconditions.checkState(!planHasInvertedJoin);

130	return null;

131

}

132

133	// we need to insert every rhs row into the hash table and then look up

134	// every lhs row

135	long lhsCardinality = root.getCardinality();

136	long rhsCardinality = minEntry.second.getCardinality();

137	numOps += lhsCardinality + rhsCardinality;

138	LOG.debug(Integer.toString(i) + " chose " + minEntry.first.getUniqueAlias()

139	+ " #lhs=" + Long.toString(lhsCardinality)

140	+ " #rhs=" + Long.toString(rhsCardinality)

141	+ " #ops=" + Long.toString(numOps));

142	remainingRefs.remove(minEntry);

143	joinedRefs.add(minEntry.first);

144	root = newRoot;

145	// assign id_ after running through the possible choices in order to end up

146	// with a dense sequence of node ids

147	root.setId(ctx_.getNextNodeId());

148	analyzer.setAssignedConjuncts(root.getAssignedConjuncts());

149

++i;

150

}

151

152	return root;

153

}

至此我们已经大概介绍了 createSingleNodePlan 的过程。
现在让我们回到 createPlan() 函数，来看看创建分布式执行计划树，即 createPlanFrangments 过程。

DistributedPlanner.createPlanFragments()
(Planner.java)
方法为单点计划树生成多个片段。具体代码如下：

1

/**

2	* 根据一些执行选项为单点计划树创建多个片段

3	* 片段通过 list 返回，list 中位置 i 的片段只能使用片段 j 的输出(j > i)。

4

*

5	* TODO: 考虑计划片段中的数据分片; 尤其是要比 createQueryPlan() 更加注重协调

6	* 聚集操作中 hash partitioning 以及分析计算中的 hash partitioning。

7	* (只有在相同 select 块中进行聚集和分析计算时才会发生协调)

8

*/

9	public ArrayList<PlanFragment> createPlanFragments(

10	PlanNode singleNodePlan) throws ImpalaException {

11	Preconditions.checkState(!ctx_.isSingleNodeExec());

12	AnalysisContext.AnalysisResult analysisResult = ctx_.getAnalysisResult();

13	QueryStmt queryStmt = ctx_.getQueryStmt();

14	ArrayList<PlanFragment> fragments = Lists.newArrayList();

15	// 对于 insert 或 CTAS，除非有 limit 限制才保持根片段 partitioned

16	// 否则，合并所有为一个单独的 coordinator fragment 以便传回到客户端

17	boolean isPartitioned = false;

18	if ((analysisResult.isInsertStmt() || analysisResult.isCreateTableAsSelectStmt())

19	&& !singleNodePlan.hasLimit()) {

20	Preconditions.checkState(!queryStmt.hasOffset());

21	isPartitioned = true;

22

}

23	LOG.debug("create plan fragments");

24	long perNodeMemLimit = ctx_.getQueryOptions().mem_limit;

25	LOG.debug("memlimit=" + Long.toString(perNodeMemLimit));

26

// 调用私有方法

27	createPlanFragments(singleNodePlan, isPartitioned, perNodeMemLimit, fragments);

28	return fragments;

29

}

上面的方法调用私有成员方法 DistributedPlanner.createPlanFragments()
DistributedPlanner.java
该方法返回生成 root 结果的 fragments。具体代码如下：

1

/**

2	* 返回生成 'root' 结果的 fragments; 递归创建所有 input fragments 到返回的 fragment

3	* 如果创建了一个新的 fragment，会被追加到 ‘fragments’，这样 fragment 就会在所有需要

4	* 它们的输出的 fragments 前面。

5	* 如果 'isPartitioned' 为否,，那么返回的 fragment 就是 unpartitioned;

6	* 否则就可能是 partitioned, 取决于它的输入是否 partitioned;

7	* the partition function is derived from the inputs.

8

*/

9	private PlanFragment createPlanFragments(

10	PlanNode root, boolean isPartitioned,

11	long perNodeMemLimit, ArrayList<PlanFragment> fragments)

12	throws InternalException, NotImplementedException {

13	ArrayList<PlanFragment> childFragments = Lists.newArrayList();

14	for (PlanNode child: root.getChildren()) {

15	// 允许子 fragments 是 partition 的，除非它们保护 limit 从句。

16	// (因为包含 limit 限制的结果集需要集中计算);

17	// 如果需要的话在后面合并

18	boolean childIsPartitioned = !child.hasLimit();

19	// 递归调用 createPlanFragments，将 child 创建的 PlanFragments 添加到 childFragments

20	childFragments.add(

21	createPlanFragments(

22	child, childIsPartitioned, perNodeMemLimit, fragments));

23

}

24	// 根据 root 的不同 Node 类型创建不同的 Fragment

25	PlanFragment result = null;

26	if (root instanceof ScanNode) {

27	result = createScanFragment(root);

28	fragments.add(result);

29	} else if (root instanceof HashJoinNode) {

30	Preconditions.checkState(childFragments.size() == 2);

31	result = createHashJoinFragment(

32	(HashJoinNode) root, childFragments.get(1), childFragments.get(0),

33	perNodeMemLimit, fragments);

34	} else if (root instanceof CrossJoinNode) {

35	Preconditions.checkState(childFragments.size() == 2);

36	result = createCrossJoinFragment(

37	(CrossJoinNode) root, childFragments.get(1), childFragments.get(0),

38	perNodeMemLimit, fragments);

39	} else if (root instanceof SelectNode) {

40	result = createSelectNodeFragment((SelectNode) root, childFragments);

41	} else if (root instanceof UnionNode) {

42	result = createUnionNodeFragment((UnionNode) root, childFragments, fragments);

43	} else if (root instanceof AggregationNode) {

44	result = createAggregationFragment(

45	(AggregationNode) root, childFragments.get(0), fragments);

46	} else if (root instanceof SortNode) {

47	if (((SortNode) root).isAnalyticSort()) {

48	// don't parallelize this like a regular SortNode

49	result = createAnalyticFragment(

50	(SortNode) root, childFragments.get(0), fragments);

51

} else {

52	result = createOrderByFragment(

53	(SortNode) root, childFragments.get(0), fragments);

54

}

55	} else if (root instanceof AnalyticEvalNode) {

56	result = createAnalyticFragment(root, childFragments.get(0), fragments);

57	} else if (root instanceof EmptySetNode) {

58	result = new PlanFragment(

59	ctx_.getNextFragmentId(), root, DataPartition.UNPARTITIONED);

60

} else {

61	throw new InternalException(

62	"Cannot create plan fragment for this node type: " + root.getExplainString());

63

}

64	// move 'result' to end, it depends on all of its children

65	fragments.remove(result);

66	fragments.add(result);

67	// 如果已经分区，还需要创建 MergeFragment

68	if (!isPartitioned && result.isPartitioned()) {

69	result = createMergeFragment(result);

70	fragments.add(result);

71

}

72

73	return result;

74

}

上面的方法调用了大量的 create*Fragment() 私有成员方法。这些成员方法的具体实现可以查看源文件：
DistributedPlanner.java

这些成员方法都返回了 PlanFragment 实例，关于该类的具体实现可以查看源代码：
PlanFragment.java

至此，我们大概介绍了 createPlanFragments 的过程。

由于 createSingleNodePlan 和 createPlanFragments 两个 createPlan 最重要的部分都已经介绍了，
createPlan 也就介绍到这里。现在让我们回到 frontend.createExecRequest()
继续来看剩下的内容。frontend.createExecRequest() 其余代码如下：

1

/**

2	* Create a populated TExecRequest corresponding to the supplied TQueryCtx.

3

*/

4	public TExecRequest createExecRequest(TQueryCtx queryCtx, StringBuilder explainString)

5	throws ImpalaException {

6

.

7

.

8

.

9

.

10

.

11

12	// 设置 fragment 的目的地

13	for (int i = 1; i < fragments.size(); ++i) {

14	PlanFragment dest = fragments.get(i).getDestFragment();

15	Integer idx = fragmentIdx.get(dest);

16	Preconditions.checkState(idx != null);

17	queryExecRequest.addToDest_fragment_idx(idx.intValue());

18

}

19

20	// 为 Scan node 设置 scan 范围/位置

21	// Also assemble list of tables names missing stats for assembling a warning message.

22	LOG.debug("get scan range locations");

23	Set<TTableName> tablesMissingStats = Sets.newTreeSet();

24	for (ScanNode scanNode: scanNodes) {

25	queryExecRequest.putToPer_node_scan_ranges(

26	scanNode.getId().asInt(),

27	scanNode.getScanRangeLocations());

28	if (scanNode.isTableMissingStats()) {

29	tablesMissingStats.add(scanNode.getTupleDesc().getTableName().toThrift());

30

}

31

}

32

// 设置主机列表

33	queryExecRequest.setHost_list(analysisResult.getAnalyzer().getHostIndex().getList());

34	for (TTableName tableName: tablesMissingStats) {

35	queryCtx.addToTables_missing_stats(tableName);

36

}

37

38	// Optionally disable spilling in the backend. Allow spilling if there are plan hints

39	// or if all tables have stats.

40	if (queryCtx.request.query_options.isDisable_unsafe_spills()

41	&& !tablesMissingStats.isEmpty()

42	&& !analysisResult.getAnalyzer().hasPlanHints()) {

43	queryCtx.setDisable_spilling(true);

44

}

45

46	// 计算资源需求，因为 scan node 的开销估计取决于这些

47

try {

48	planner.computeResourceReqs(fragments, true, queryExecRequest);

49	} catch (Exception e) {

50	// 将异常转换为警告，以便查询能继续执行

51	LOG.error("Failed to compute resource requirements for query " +

52	queryCtx.request.getStmt(), e);

53

}

54

55	// 到了这里 fragment 所有信息都设置好了，序列化到 Thrift

56	for (PlanFragment fragment: fragments) {

57	TPlanFragment thriftFragment = fragment.toThrift();

58	queryExecRequest.addToFragments(thriftFragment);

59

}

60

61	// Use VERBOSE by default for all non-explain statements.

62	TExplainLevel explainLevel = TExplainLevel.VERBOSE;

63	// Use the query option for explain stmts and tests (e.g., planner tests).

64	if (analysisResult.isExplainStmt() || RuntimeEnv.INSTANCE.isTestEnv()) {

65	explainLevel = queryCtx.request.query_options.getExplain_level();

66

}

67

68	// Global query parameters to be set in each TPlanExecRequest.

69	queryExecRequest.setQuery_ctx(queryCtx);

70

71	explainString.append(

72	planner.getExplainString(fragments, queryExecRequest, explainLevel));

73	queryExecRequest.setQuery_plan(explainString.toString());

74	queryExecRequest.setDesc_tbl(analysisResult.getAnalyzer().getDescTbl().toThrift());

75

76	String jsonLineageGraph = analysisResult.getJsonLineageGraph();

77	if (jsonLineageGraph != null && !jsonLineageGraph.isEmpty()) {

78	queryExecRequest.setLineage_graph(jsonLineageGraph);

79

}

80

81	if (analysisResult.isExplainStmt()) {

82	// Return the EXPLAIN request

83	createExplainRequest(explainString.toString(), result);

84	return result;

85

}

86

87	result.setQuery_exec_request(queryExecRequest);

88

89	if (analysisResult.isQueryStmt()) {

90

// 填充元数据

91	LOG.debug("create result set metadata");

92	result.stmt_type = TStmtType.QUERY;

93	result.query_exec_request.stmt_type = result.stmt_type;

94	TResultSetMetadata metadata = new TResultSetMetadata();

95	QueryStmt queryStmt = analysisResult.getQueryStmt();

96	int colCnt = queryStmt.getColLabels().size();

97	for (int i = 0; i < colCnt; ++i) {

98	TColumn colDesc = new TColumn();

99	colDesc.columnName = queryStmt.getColLabels().get(i);

100	colDesc.columnType = queryStmt.getResultExprs().get(i).getType().toThrift();

101	metadata.addToColumns(colDesc);

102

}

103	result.setResult_set_metadata(metadata);

104

} else {

105	Preconditions.checkState(analysisResult.isInsertStmt() ||

106	analysisResult.isCreateTableAsSelectStmt());

107

108	// For CTAS the overall TExecRequest statement type is DDL, but the

109	// query_exec_request should be DML

110	result.stmt_type =

111	analysisResult.isCreateTableAsSelectStmt() ? TStmtType.DDL : TStmtType.DML;

112	result.query_exec_request.stmt_type = TStmtType.DML;

113

114	// create finalization params of insert stmt

115	InsertStmt insertStmt = analysisResult.getInsertStmt();

116	if (insertStmt.getTargetTable() instanceof HdfsTable) {

117	TFinalizeParams finalizeParams = new TFinalizeParams();

118	finalizeParams.setIs_overwrite(insertStmt.isOverwrite());

119	finalizeParams.setTable_name(insertStmt.getTargetTableName().getTbl());

120	finalizeParams.setTable_id(insertStmt.getTargetTable().getId().asInt());

121	String db = insertStmt.getTargetTableName().getDb();

122	finalizeParams.setTable_db(db == null ? queryCtx.session.database : db);

123	HdfsTable hdfsTable = (HdfsTable) insertStmt.getTargetTable();

124	finalizeParams.setHdfs_base_dir(hdfsTable.getHdfsBaseDir());

125	finalizeParams.setStaging_dir(

126	hdfsTable.getHdfsBaseDir() + "/_impala_insert_staging");

127	queryExecRequest.setFinalize_params(finalizeParams);

128

}

129

}

130

131	validateTableIds(analysisResult.getAnalyzer(), result);

132

133	timeline.markEvent("Planning finished");

134	result.setTimeline(analysisResult.getAnalyzer().getTimeline().toThrift());

135	return result;

136

}

至此，FE 结束，返回 TExecRequest 型的对象给 backend 执行。

由于笔者刚开始接触 Impala，分析可能存在某些谬误，有任何疑问或建议都欢迎讨论。

---