jpullokkaran / hive-cbo-apache-trunk Goto Github PK

Exception: pre: !rel.getTraits().equals(toTraits)
java.lang.AssertionError: pre: !rel.getTraits().equals(toTraits)
at org.eigenbase.relopt.volcano.VolcanoPlanner.changeTraits(VolcanoPlanner.java:490)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:135)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:39)
at net.hydromatic.optiq.prepare.OptiqPrepareImpl.withPlanner(OptiqPrepareImpl.java:508)
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:91)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.optimize(CBO.java:64)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:8962)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:64)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:444)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:344)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1050)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1093)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:986)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:976)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:268)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:220)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:424)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:359)
at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:922)
at org.apache.hadoop.hive.cli.TestCliDriver.runTest(TestCliDriver.java:133)
at org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver_join22_cbo(TestCliDriver.java:117)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at junit.framework.TestCase.runTest(TestCase.java:168)
at junit.framework.TestCase.runBare(TestCase.java:134)
at junit.framework.TestResult$1.protect(TestResult.java:110)
at junit.framework.TestResult.runProtected(TestResult.java:128)
at junit.framework.TestResult.run(TestResult.java:113)
at junit.framework.TestCase.run(TestCase.java:124)
at junit.framework.TestSuite.runTest(TestSuite.java:243)
at junit.framework.TestSuite.run(TestSuite.java:238)
at org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:83)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:264)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:124)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
Failed query: join22_cbo.q

Following query:
explain
select r1.i, r2.i, r3.i
from over10k r1 join over5k r2 join over500 r3
on r1.d = r2.d and r1.d = r3.d and
r1.f = r2.f and r1.f = r3.f and
r1.bo = r2.bo and r1.bo = r3.bo and
r1.i = r2.i and r1.i = r3.i
where r1.i + r2.i + r3.i = (65536) * 3
;

throws IndexOutOfBoundsException.

See cbo_datatypes.q on how to setup schema and data for this test

StackTrace is:
java.lang.RuntimeException: java.lang.IndexOutOfBoundsException: index (5) must be less than size (4)
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:93)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.optimize(CBO.java:64)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:8971)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:64)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:444)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:344)
....

explain select tbl1.a_int from tbl1 join tbl2 on tbl1.a_int=tbl2.a_int order by tbl1.a_int;

explain select * from (select * from tbl1 where (tbl1.a_int+tbl1.c_float) > 1)r1 join (select * from tbl2 where (tbl2.c_float *10) > 1)r2 on r1.a_int=r2.a_int join tbl3 on r1.a_int=tbl3.a_int join tbl4 on r1.a_int=tbl4.a_int;

ava.lang.RuntimeException: java.lang.NullPointerException
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:100)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.optimize(CBO.java:69)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:8981)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:64)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:444)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:344)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1040)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1083)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:976)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:966)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:268)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:220)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:424)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:359)
at org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:457)
at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:467)
at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:748)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:686)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:625)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:160)
Caused by: java.lang.NullPointerException
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter$Context.columnPosition(RelNodeConverter.java:228)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter$Context.buildColumnMap(RelNodeConverter.java:198)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter$JoinProcessor.process(RelNodeConverter.java:272)
at org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatch(DefaultGraphWalker.java:78)
at org.apache.hadoop.hive.ql.optimizer.optiq.ForwardWalker.walk(ForwardWalker.java:60)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(DefaultGraphWalker.java:109)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter.convert(RelNodeConverter.java:109)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:108)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:37)
at net.hydromatic.optiq.prepare.OptiqPrepareImpl.withPlanner(OptiqPrepareImpl.java:508)
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:98)
... 25 more

To run from itests/qtest run:
mvn test -Dtest=TestCliDriver -Dqfile=join12_cbo.q -Phadoop-1

StackTrace in:
itests/qtest/target/surefire-reports/org.apache.hadoop.hive.cli.TestCliDriver-output.txt

Sample trace is:
java.lang.RuntimeException: java.lang.UnsupportedOperationException
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:93)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.optimize(CBO.java:64)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:8962)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:64)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:444)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:344)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1050)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1093)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:986)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:976)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:268)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:220)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:424)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:359)
at org.apache.hadoop.hive.ql.QTestUtil.executeClient(QTestUtil.java:922)
at org.apache.hadoop.hive.cli.TestCliDriver.runTest(TestCliDriver.java:133)
at org.apache.hadoop.hive.cli.TestCliDriver.testCliDriver_join12_cbo(TestCliDriver.java:117)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at junit.framework.TestCase.runTest(TestCase.java:168)
at junit.framework.TestCase.runBare(TestCase.java:134)
at junit.framework.TestResult$1.protect(TestResult.java:110)
at junit.framework.TestResult.runProtected(TestResult.java:128)
at junit.framework.TestResult.run(TestResult.java:113)
at junit.framework.TestCase.run(TestCase.java:124)
at junit.framework.TestSuite.runTest(TestSuite.java:243)
at junit.framework.TestSuite.run(TestSuite.java:238)
at org.junit.internal.runners.JUnit38ClassRunner.run(JUnit38ClassRunner.java:83)
at org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:264)
at org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:153)
at org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:124)
at org.apache.maven.surefire.booter.ForkedBooter.invokeProviderInSameClassLoader(ForkedBooter.java:200)
at org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:153)
at org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:103)
Caused by: java.lang.UnsupportedOperationException
at java.util.AbstractList.add(AbstractList.java:131)
at java.util.AbstractList.add(AbstractList.java:91)
at org.apache.hadoop.hive.ql.optimizer.optiq.OptiqTraitsUtil.getSingleRelTraitSet(OptiqTraitsUtil.java:505)
at org.apache.hadoop.hive.ql.optimizer.optiq.OptiqTraitsUtil.getFilterTraitSet(OptiqTraitsUtil.java:217)
at org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveFilterRel.(HiveFilterRel.java:26)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertFilterOp(HiveToOptiqRelConverter.java:211)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOp(HiveToOptiqRelConverter.java:171)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:154)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:147)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:147)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:147)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:147)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:147)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:147)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:147)
at org.apache.hadoop.hive.ql.optimizer.optiq.HiveToOptiqRelConverter.convertOpDAG(HiveToOptiqRelConverter.java:423)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:102)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:39)
at net.hydromatic.optiq.prepare.OptiqPrepareImpl.withPlanner(OptiqPrepareImpl.java:508)
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:91)
... 37 more

if you run the cbo* tests with -Dhive.cbo.enable=true you will see the following stacktraces. This is becuase cbo code path is attempted for the analyze commands. Took me hours to track this down...

Caused by: java.lang.RuntimeException: Incomplete Col stats
at org.apache.hadoop.hive.ql.optimizer.optiq.stats.OptiqStatsUtil.computeTableRelColStat(OptiqStatsUtil.java:305)
at org.apache.hadoop.hive.ql.optimizer.optiq.reloperators.HiveTableScanRel.(HiveTableScanRel.java:66)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter$TableScanProcessor.process(RelNodeConverter.java:585)
at org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatch(DefaultGraphWalker.java:78)
at org.apache.hadoop.hive.ql.optimizer.optiq.ForwardWalker.walk(ForwardWalker.java:60)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(DefaultGraphWalker.java:109)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter.convert(RelNodeConverter.java:109)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:101)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:1)
at net.hydromatic.optiq.prepare.OptiqPrepareImpl.withPlanner(OptiqPrepareImpl.java:508)
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:98)

All the tests in joins_cbo.q fail at this step now.

(Filing for Deepesh)

Query is:
select name, contributions, max(contributions) over (partition by registration) from votertab10k;

See following in log
14/02/21 22:16:34 INFO parse.SemanticAnalyzer: CBO failed, skipping CBO

explain select * from t1 join t2 on ((t1.x+t1.y)/10)=(t2.x+t2.y*5);
Fails in conversion.
java.lang.RuntimeException: java.lang.NullPointerException
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:100)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.optimize(CBO.java:68)
at org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.analyzeInternal(SemanticAnalyzer.java:8981)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.parse.ExplainSemanticAnalyzer.analyzeInternal(ExplainSemanticAnalyzer.java:64)
at org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.analyze(BaseSemanticAnalyzer.java:310)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:444)
at org.apache.hadoop.hive.ql.Driver.compile(Driver.java:344)
at org.apache.hadoop.hive.ql.Driver.compileInternal(Driver.java:1040)
at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1083)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:976)
at org.apache.hadoop.hive.ql.Driver.run(Driver.java:966)
at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:268)
at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:220)
at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:424)
at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:792)
at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:686)
at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:625)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:160)
Caused by: java.lang.NullPointerException
at org.eigenbase.rex.RexBuilder.deriveReturnType(RexBuilder.java:259)
at org.eigenbase.rex.RexBuilder.makeCall(RexBuilder.java:218)
at org.apache.hadoop.hive.ql.optimizer.optiq.expr.RexNodeConverter.convert(RexNodeConverter.java:59)
at org.apache.hadoop.hive.ql.optimizer.optiq.expr.RexNodeConverter.convert(RexNodeConverter.java:40)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter$Context.convertToOptiqExpr(RelNodeConverter.java:221)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter$JoinProcessor.convertJoinOp(RelNodeConverter.java:289)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter$JoinProcessor.process(RelNodeConverter.java:242)
at org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher.dispatch(DefaultRuleDispatcher.java:90)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatchAndReturn(DefaultGraphWalker.java:94)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.dispatch(DefaultGraphWalker.java:78)
at org.apache.hadoop.hive.ql.optimizer.optiq.ForwardWalker.walk(ForwardWalker.java:60)
at org.apache.hadoop.hive.ql.lib.DefaultGraphWalker.startWalking(DefaultGraphWalker.java:109)
at org.apache.hadoop.hive.ql.optimizer.optiq.RelNodeConverter.convert(RelNodeConverter.java:87)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:107)
at org.apache.hadoop.hive.ql.optimizer.optiq.CBO.apply(CBO.java:41)
at net.hydromatic.optiq.prepare.OptiqPrepareImpl.withPlanner(OptiqPrepareImpl.java:508)
at net.hydromatic.optiq.tools.Frameworks.withPlanner(Frameworks.java:98)