== Physical Plan ==
CollectLimit (11)
+- * ColumnarToRow (10)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (9)
+- * Filter (8)
+- * ColumnarToRow (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [7]: [yield#94085910, volatility#94085907, momentum#94085909, size#94085878, value#94085879, growth#94085880, leverage#94085881]
Arguments: [yield#94085910, volatility#94085907, momentum#94085909, size#94085878, value#94085879, growth#94085880, leverage#94085881]
(2) InMemoryRelation
Arguments: [yield#94085910, volatility#94085907, momentum#94085909, size#94085878, value#94085879, growth#94085880, leverage#94085881], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [yield#94085910, volatility#94085907, momentum#94085909, size#94085878, value#94085879, growth#94085880, leverage#94085881]
+- *(1) Filter (isnotnull(cap#94085875) AND (cap#94085875 = 0.0))
+- *(1) ColumnarToRow
+- InMemoryTableScan [cap#94085875, growth#94085880, leverage#94085881, momentum#94085909, size#94085878, value#94085879, volatility#94085907, yield#94085910], [isnotnull(cap#94085875), (cap#94085875 = 0.0)]
+- InMemoryRelation [overall#94085874, cap#94085875, retIC#94085876, resretIC#94085877, size#94085878, value#94085879, growth#94085880, leverage#94085881, volatility#94085907, momentum#94085909, yield#94085910, numcos#94085911, numdates#94085912, annual_bmret#94085916, annual_ret#94085917, std_ret#94085918, Sharpe_ret#94085919, PctPos_ret#94085920, TR_ret#94085921, IR_ret#94085922, annual_resret#94085923, std_resret#94085924, Sharpe_resret#94085925, PctPos_resret#94085926, ... 9 more fields], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN ((overall#94085727 = NA) OR (overall#94085727 = null)) THEN null ELSE cast(overall#94085727 as int) END AS overall#94085874, CASE WHEN ((cap#94085728 = NA) OR (cap#94085728 = null)) THEN null ELSE cast(cap#94085728 as float) END AS cap#94085875, CASE WHEN ((retIC#94085729 = NA) OR (retIC#94085729 = null)) THEN null ELSE cast(retIC#94085729 as float) END AS retIC#94085876, CASE WHEN ((resretIC#94085730 = NA) OR (resretIC#94085730 = null)) THEN null ELSE cast(resretIC#94085730 as float) END AS resretIC#94085877, CASE WHEN ((size#94085731 = NA) OR (size#94085731 = null)) THEN null ELSE cast(size#94085731 as float) END AS size#94085878, CASE WHEN ((value#94085732 = NA) OR (value#94085732 = null)) THEN null ELSE cast(value#94085732 as float) END AS value#94085879, CASE WHEN ((growth#94085733 = NA) OR (growth#94085733 = null)) THEN null ELSE cast(growth#94085733 as float) END AS growth#94085880, CASE WHEN ((leverage#94085734 = NA) OR (leverage#94085734 = null)) THEN null ELSE cast(leverage#94085734 as float) END AS leverage#94085881, CASE WHEN ((volatility#94085735 = NA) OR (volatility#94085735 = null)) THEN null ELSE cast(volatility#94085735 as float) END AS volatility#94085907, CASE WHEN ((momentum#94085736 = NA) OR (momentum#94085736 = null)) THEN null ELSE cast(momentum#94085736 as float) END AS momentum#94085909, CASE WHEN ((yield#94085737 = NA) OR (yield#94085737 = null)) THEN null ELSE cast(yield#94085737 as float) END AS yield#94085910, CASE WHEN ((numcos#94085738 = NA) OR (numcos#94085738 = null)) THEN null ELSE cast(numcos#94085738 as float) END AS numcos#94085911, CASE WHEN ((numdates#94085739 = NA) OR (numdates#94085739 = null)) THEN null ELSE cast(numdates#94085739 as float) END AS numdates#94085912, CASE WHEN ((annual_bmret#94085740 = NA) OR (annual_bmret#94085740 = null)) THEN null ELSE cast(annual_bmret#94085740 as float) END AS annual_bmret#94085916, CASE WHEN ((annual_ret#94085741 = NA) OR (annual_ret#94085741 = null)) THEN null ELSE cast(annual_ret#94085741 as float) END AS annual_ret#94085917, CASE WHEN ((std_ret#94085742 = NA) OR (std_ret#94085742 = null)) THEN null ELSE cast(std_ret#94085742 as float) END AS std_ret#94085918, CASE WHEN ((Sharpe_ret#94085743 = NA) OR (Sharpe_ret#94085743 = null)) THEN null ELSE cast(Sharpe_ret#94085743 as float) END AS Sharpe_ret#94085919, CASE WHEN ((PctPos_ret#94085744 = NA) OR (PctPos_ret#94085744 = null)) THEN null ELSE cast(PctPos_ret#94085744 as float) END AS PctPos_ret#94085920, CASE WHEN ((TR_ret#94085745 = NA) OR (TR_ret#94085745 = null)) THEN null ELSE cast(TR_ret#94085745 as float) END AS TR_ret#94085921, CASE WHEN ((IR_ret#94085746 = NA) OR (IR_ret#94085746 = null)) THEN null ELSE cast(IR_ret#94085746 as float) END AS IR_ret#94085922, CASE WHEN ((annual_resret#94085747 = NA) OR (annual_resret#94085747 = null)) THEN null ELSE cast(annual_resret#94085747 as float) END AS annual_resret#94085923, CASE WHEN ((std_resret#94085748 = NA) OR (std_resret#94085748 = null)) THEN null ELSE cast(std_resret#94085748 as float) END AS std_resret#94085924, CASE WHEN ((Sharpe_resret#94085749 = NA) OR (Sharpe_resret#94085749 = null)) THEN null ELSE cast(Sharpe_resret#94085749 as float) END AS Sharpe_resret#94085925, CASE WHEN ((PctPos_resret#94085750 = NA) OR (PctPos_resret#94085750 = null)) THEN null ELSE cast(PctPos_resret#94085750 as float) END AS PctPos_resret#94085926, ... 9 more fields]
+- FileScan csv [overall#94085727,cap#94085728,retIC#94085729,resretIC#94085730,size#94085731,value#94085732,growth#94085733,leverage#94085734,volatility#94085735,momentum#94085736,yield#94085737,numcos#94085738,numdates#94085739,annual_bmret#94085740,annual_ret#94085741,std_ret#94085742,Sharpe_ret#94085743,PctPos_ret#94085744,TR_ret#94085745,IR_ret#94085746,annual_resret#94085747,std_resret#94085748,Sharpe_resret#94085749,PctPos_resret#94085750,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/volatilit..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(3) InMemoryTableScan
Output [8]: [cap#94085875, growth#94085880, leverage#94085881, momentum#94085909, size#94085878, value#94085879, volatility#94085907, yield#94085910]
Arguments: [cap#94085875, growth#94085880, leverage#94085881, momentum#94085909, size#94085878, value#94085879, volatility#94085907, yield#94085910], [isnotnull(cap#94085875), (cap#94085875 = 0.0)]
(4) InMemoryRelation
Arguments: [overall#94085874, cap#94085875, retIC#94085876, resretIC#94085877, size#94085878, value#94085879, growth#94085880, leverage#94085881, volatility#94085907, momentum#94085909, yield#94085910, numcos#94085911, numdates#94085912, annual_bmret#94085916, annual_ret#94085917, std_ret#94085918, Sharpe_ret#94085919, PctPos_ret#94085920, TR_ret#94085921, IR_ret#94085922, annual_resret#94085923, std_resret#94085924, Sharpe_resret#94085925, PctPos_resret#94085926, ... 9 more fields], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((overall#94085727 = NA) OR (overall#94085727 = null)) THEN null ELSE cast(overall#94085727 as int) END AS overall#94085874, CASE WHEN ((cap#94085728 = NA) OR (cap#94085728 = null)) THEN null ELSE cast(cap#94085728 as float) END AS cap#94085875, CASE WHEN ((retIC#94085729 = NA) OR (retIC#94085729 = null)) THEN null ELSE cast(retIC#94085729 as float) END AS retIC#94085876, CASE WHEN ((resretIC#94085730 = NA) OR (resretIC#94085730 = null)) THEN null ELSE cast(resretIC#94085730 as float) END AS resretIC#94085877, CASE WHEN ((size#94085731 = NA) OR (size#94085731 = null)) THEN null ELSE cast(size#94085731 as float) END AS size#94085878, CASE WHEN ((value#94085732 = NA) OR (value#94085732 = null)) THEN null ELSE cast(value#94085732 as float) END AS value#94085879, CASE WHEN ((growth#94085733 = NA) OR (growth#94085733 = null)) THEN null ELSE cast(growth#94085733 as float) END AS growth#94085880, CASE WHEN ((leverage#94085734 = NA) OR (leverage#94085734 = null)) THEN null ELSE cast(leverage#94085734 as float) END AS leverage#94085881, CASE WHEN ((volatility#94085735 = NA) OR (volatility#94085735 = null)) THEN null ELSE cast(volatility#94085735 as float) END AS volatility#94085907, CASE WHEN ((momentum#94085736 = NA) OR (momentum#94085736 = null)) THEN null ELSE cast(momentum#94085736 as float) END AS momentum#94085909, CASE WHEN ((yield#94085737 = NA) OR (yield#94085737 = null)) THEN null ELSE cast(yield#94085737 as float) END AS yield#94085910, CASE WHEN ((numcos#94085738 = NA) OR (numcos#94085738 = null)) THEN null ELSE cast(numcos#94085738 as float) END AS numcos#94085911, CASE WHEN ((numdates#94085739 = NA) OR (numdates#94085739 = null)) THEN null ELSE cast(numdates#94085739 as float) END AS numdates#94085912, CASE WHEN ((annual_bmret#94085740 = NA) OR (annual_bmret#94085740 = null)) THEN null ELSE cast(annual_bmret#94085740 as float) END AS annual_bmret#94085916, CASE WHEN ((annual_ret#94085741 = NA) OR (annual_ret#94085741 = null)) THEN null ELSE cast(annual_ret#94085741 as float) END AS annual_ret#94085917, CASE WHEN ((std_ret#94085742 = NA) OR (std_ret#94085742 = null)) THEN null ELSE cast(std_ret#94085742 as float) END AS std_ret#94085918, CASE WHEN ((Sharpe_ret#94085743 = NA) OR (Sharpe_ret#94085743 = null)) THEN null ELSE cast(Sharpe_ret#94085743 as float) END AS Sharpe_ret#94085919, CASE WHEN ((PctPos_ret#94085744 = NA) OR (PctPos_ret#94085744 = null)) THEN null ELSE cast(PctPos_ret#94085744 as float) END AS PctPos_ret#94085920, CASE WHEN ((TR_ret#94085745 = NA) OR (TR_ret#94085745 = null)) THEN null ELSE cast(TR_ret#94085745 as float) END AS TR_ret#94085921, CASE WHEN ((IR_ret#94085746 = NA) OR (IR_ret#94085746 = null)) THEN null ELSE cast(IR_ret#94085746 as float) END AS IR_ret#94085922, CASE WHEN ((annual_resret#94085747 = NA) OR (annual_resret#94085747 = null)) THEN null ELSE cast(annual_resret#94085747 as float) END AS annual_resret#94085923, CASE WHEN ((std_resret#94085748 = NA) OR (std_resret#94085748 = null)) THEN null ELSE cast(std_resret#94085748 as float) END AS std_resret#94085924, CASE WHEN ((Sharpe_resret#94085749 = NA) OR (Sharpe_resret#94085749 = null)) THEN null ELSE cast(Sharpe_resret#94085749 as float) END AS Sharpe_resret#94085925, CASE WHEN ((PctPos_resret#94085750 = NA) OR (PctPos_resret#94085750 = null)) THEN null ELSE cast(PctPos_resret#94085750 as float) END AS PctPos_resret#94085926, ... 9 more fields]
+- FileScan csv [overall#94085727,cap#94085728,retIC#94085729,resretIC#94085730,size#94085731,value#94085732,growth#94085733,leverage#94085734,volatility#94085735,momentum#94085736,yield#94085737,numcos#94085738,numdates#94085739,annual_bmret#94085740,annual_ret#94085741,std_ret#94085742,Sharpe_ret#94085743,PctPos_ret#94085744,TR_ret#94085745,IR_ret#94085746,annual_resret#94085747,std_resret#94085748,Sharpe_resret#94085749,PctPos_resret#94085750,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/volatilit..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(5) Scan csv
Output [33]: [overall#94085727, cap#94085728, retIC#94085729, resretIC#94085730, size#94085731, value#94085732, growth#94085733, leverage#94085734, volatility#94085735, momentum#94085736, yield#94085737, numcos#94085738, numdates#94085739, annual_bmret#94085740, annual_ret#94085741, std_ret#94085742, Sharpe_ret#94085743, PctPos_ret#94085744, TR_ret#94085745, IR_ret#94085746, annual_resret#94085747, std_resret#94085748, Sharpe_resret#94085749, PctPos_resret#94085750, TR_resret#94085751, IR_resret#94085752, annual_retnet#94085753, std_retnet#94085754, Sharpe_retnet#94085755, PctPos_retnet#94085756, TR_retnet#94085757, IR_retnet#94085758, turnover#94085759]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/volatility/stats_overall.csv]
ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:string,leverage:string,volatility:string,momentum:string,yield:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string>
(6) Project [codegen id : 1]
Output [33]: [CASE WHEN ((overall#94085727 = NA) OR (overall#94085727 = null)) THEN null ELSE cast(overall#94085727 as int) END AS overall#94085874, CASE WHEN ((cap#94085728 = NA) OR (cap#94085728 = null)) THEN null ELSE cast(cap#94085728 as float) END AS cap#94085875, CASE WHEN ((retIC#94085729 = NA) OR (retIC#94085729 = null)) THEN null ELSE cast(retIC#94085729 as float) END AS retIC#94085876, CASE WHEN ((resretIC#94085730 = NA) OR (resretIC#94085730 = null)) THEN null ELSE cast(resretIC#94085730 as float) END AS resretIC#94085877, CASE WHEN ((size#94085731 = NA) OR (size#94085731 = null)) THEN null ELSE cast(size#94085731 as float) END AS size#94085878, CASE WHEN ((value#94085732 = NA) OR (value#94085732 = null)) THEN null ELSE cast(value#94085732 as float) END AS value#94085879, CASE WHEN ((growth#94085733 = NA) OR (growth#94085733 = null)) THEN null ELSE cast(growth#94085733 as float) END AS growth#94085880, CASE WHEN ((leverage#94085734 = NA) OR (leverage#94085734 = null)) THEN null ELSE cast(leverage#94085734 as float) END AS leverage#94085881, CASE WHEN ((volatility#94085735 = NA) OR (volatility#94085735 = null)) THEN null ELSE cast(volatility#94085735 as float) END AS volatility#94085907, CASE WHEN ((momentum#94085736 = NA) OR (momentum#94085736 = null)) THEN null ELSE cast(momentum#94085736 as float) END AS momentum#94085909, CASE WHEN ((yield#94085737 = NA) OR (yield#94085737 = null)) THEN null ELSE cast(yield#94085737 as float) END AS yield#94085910, CASE WHEN ((numcos#94085738 = NA) OR (numcos#94085738 = null)) THEN null ELSE cast(numcos#94085738 as float) END AS numcos#94085911, CASE WHEN ((numdates#94085739 = NA) OR (numdates#94085739 = null)) THEN null ELSE cast(numdates#94085739 as float) END AS numdates#94085912, CASE WHEN ((annual_bmret#94085740 = NA) OR (annual_bmret#94085740 = null)) THEN null ELSE cast(annual_bmret#94085740 as float) END AS annual_bmret#94085916, CASE WHEN ((annual_ret#94085741 = NA) OR (annual_ret#94085741 = null)) THEN null ELSE cast(annual_ret#94085741 as float) END AS annual_ret#94085917, CASE WHEN ((std_ret#94085742 = NA) OR (std_ret#94085742 = null)) THEN null ELSE cast(std_ret#94085742 as float) END AS std_ret#94085918, CASE WHEN ((Sharpe_ret#94085743 = NA) OR (Sharpe_ret#94085743 = null)) THEN null ELSE cast(Sharpe_ret#94085743 as float) END AS Sharpe_ret#94085919, CASE WHEN ((PctPos_ret#94085744 = NA) OR (PctPos_ret#94085744 = null)) THEN null ELSE cast(PctPos_ret#94085744 as float) END AS PctPos_ret#94085920, CASE WHEN ((TR_ret#94085745 = NA) OR (TR_ret#94085745 = null)) THEN null ELSE cast(TR_ret#94085745 as float) END AS TR_ret#94085921, CASE WHEN ((IR_ret#94085746 = NA) OR (IR_ret#94085746 = null)) THEN null ELSE cast(IR_ret#94085746 as float) END AS IR_ret#94085922, CASE WHEN ((annual_resret#94085747 = NA) OR (annual_resret#94085747 = null)) THEN null ELSE cast(annual_resret#94085747 as float) END AS annual_resret#94085923, CASE WHEN ((std_resret#94085748 = NA) OR (std_resret#94085748 = null)) THEN null ELSE cast(std_resret#94085748 as float) END AS std_resret#94085924, CASE WHEN ((Sharpe_resret#94085749 = NA) OR (Sharpe_resret#94085749 = null)) THEN null ELSE cast(Sharpe_resret#94085749 as float) END AS Sharpe_resret#94085925, CASE WHEN ((PctPos_resret#94085750 = NA) OR (PctPos_resret#94085750 = null)) THEN null ELSE cast(PctPos_resret#94085750 as float) END AS PctPos_resret#94085926, CASE WHEN ((TR_resret#94085751 = NA) OR (TR_resret#94085751 = null)) THEN null ELSE cast(TR_resret#94085751 as float) END AS TR_resret#94085927, CASE WHEN ((IR_resret#94085752 = NA) OR (IR_resret#94085752 = null)) THEN null ELSE cast(IR_resret#94085752 as float) END AS IR_resret#94085928, CASE WHEN ((annual_retnet#94085753 = NA) OR (annual_retnet#94085753 = null)) THEN null ELSE cast(annual_retnet#94085753 as float) END AS annual_retnet#94085929, CASE WHEN ((std_retnet#94085754 = NA) OR (std_retnet#94085754 = null)) THEN null ELSE cast(std_retnet#94085754 as float) END AS std_retnet#94085930, CASE WHEN ((Sharpe_retnet#94085755 = NA) OR (Sharpe_retnet#94085755 = null)) THEN null ELSE cast(Sharpe_retnet#94085755 as float) END AS Sharpe_retnet#94085931, CASE WHEN ((PctPos_retnet#94085756 = NA) OR (PctPos_retnet#94085756 = null)) THEN null ELSE cast(PctPos_retnet#94085756 as float) END AS PctPos_retnet#94085932, CASE WHEN ((TR_retnet#94085757 = NA) OR (TR_retnet#94085757 = null)) THEN null ELSE cast(TR_retnet#94085757 as float) END AS TR_retnet#94085933, CASE WHEN ((IR_retnet#94085758 = NA) OR (IR_retnet#94085758 = null)) THEN null ELSE cast(IR_retnet#94085758 as float) END AS IR_retnet#94085934, CASE WHEN ((turnover#94085759 = NA) OR (turnover#94085759 = null)) THEN null ELSE cast(turnover#94085759 as float) END AS turnover#94085935]
Input [33]: [overall#94085727, cap#94085728, retIC#94085729, resretIC#94085730, size#94085731, value#94085732, growth#94085733, leverage#94085734, volatility#94085735, momentum#94085736, yield#94085737, numcos#94085738, numdates#94085739, annual_bmret#94085740, annual_ret#94085741, std_ret#94085742, Sharpe_ret#94085743, PctPos_ret#94085744, TR_ret#94085745, IR_ret#94085746, annual_resret#94085747, std_resret#94085748, Sharpe_resret#94085749, PctPos_resret#94085750, TR_resret#94085751, IR_resret#94085752, annual_retnet#94085753, std_retnet#94085754, Sharpe_retnet#94085755, PctPos_retnet#94085756, TR_retnet#94085757, IR_retnet#94085758, turnover#94085759]
(7) ColumnarToRow [codegen id : 1]
Input [8]: [cap#94085875, growth#94085880, leverage#94085881, momentum#94085909, size#94085878, value#94085879, volatility#94085907, yield#94085910]
(8) Filter [codegen id : 1]
Input [8]: [cap#94085875, growth#94085880, leverage#94085881, momentum#94085909, size#94085878, value#94085879, volatility#94085907, yield#94085910]
Condition : (isnotnull(cap#94085875) AND (cap#94085875 = 0.0))
(9) Project [codegen id : 1]
Output [7]: [yield#94085910, volatility#94085907, momentum#94085909, size#94085878, value#94085879, growth#94085880, leverage#94085881]
Input [8]: [cap#94085875, growth#94085880, leverage#94085881, momentum#94085909, size#94085878, value#94085879, volatility#94085907, yield#94085910]
(10) ColumnarToRow [codegen id : 1]
Input [7]: [yield#94085910, volatility#94085907, momentum#94085909, size#94085878, value#94085879, growth#94085880, leverage#94085881]
(11) CollectLimit
Input [7]: [yield#94085910, volatility#94085907, momentum#94085909, size#94085878, value#94085879, growth#94085880, leverage#94085881]
Arguments: 1000000