== Physical Plan ==
CollectLimit (11)
+- * ColumnarToRow (10)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (9)
+- * Filter (8)
+- * ColumnarToRow (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [7]: [yield#94042740, volatility#94042736, momentum#94042738, size#94042727, value#94042729, growth#94042731, leverage#94042734]
Arguments: [yield#94042740, volatility#94042736, momentum#94042738, size#94042727, value#94042729, growth#94042731, leverage#94042734]
(2) InMemoryRelation
Arguments: [yield#94042740, volatility#94042736, momentum#94042738, size#94042727, value#94042729, growth#94042731, leverage#94042734], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [yield#94042740, volatility#94042736, momentum#94042738, size#94042727, value#94042729, growth#94042731, leverage#94042734]
+- *(1) Filter (isnotnull(cap#94042720) AND (cap#94042720 = 0.0))
+- *(1) ColumnarToRow
+- InMemoryTableScan [cap#94042720, growth#94042731, leverage#94042734, momentum#94042738, size#94042727, value#94042729, volatility#94042736, yield#94042740], [isnotnull(cap#94042720), (cap#94042720 = 0.0)]
+- InMemoryRelation [overall#94042718, cap#94042720, retIC#94042722, resretIC#94042725, size#94042727, value#94042729, growth#94042731, leverage#94042734, volatility#94042736, momentum#94042738, yield#94042740, numcos#94042742, numdates#94042745, annual_bmret#94042747, annual_ret#94042748, std_ret#94042750, Sharpe_ret#94042751, PctPos_ret#94042752, TR_ret#94042753, IR_ret#94042754, annual_resret#94042755, std_resret#94042756, Sharpe_resret#94042757, PctPos_resret#94042758, ... 9 more fields], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN ((overall#94042528 = NA) OR (overall#94042528 = null)) THEN null ELSE cast(overall#94042528 as int) END AS overall#94042718, CASE WHEN ((cap#94042529 = NA) OR (cap#94042529 = null)) THEN null ELSE cast(cap#94042529 as float) END AS cap#94042720, CASE WHEN ((retIC#94042530 = NA) OR (retIC#94042530 = null)) THEN null ELSE cast(retIC#94042530 as float) END AS retIC#94042722, CASE WHEN ((resretIC#94042531 = NA) OR (resretIC#94042531 = null)) THEN null ELSE cast(resretIC#94042531 as float) END AS resretIC#94042725, CASE WHEN ((size#94042532 = NA) OR (size#94042532 = null)) THEN null ELSE cast(size#94042532 as float) END AS size#94042727, CASE WHEN ((value#94042533 = NA) OR (value#94042533 = null)) THEN null ELSE cast(value#94042533 as float) END AS value#94042729, CASE WHEN ((growth#94042534 = NA) OR (growth#94042534 = null)) THEN null ELSE cast(growth#94042534 as float) END AS growth#94042731, CASE WHEN ((leverage#94042535 = NA) OR (leverage#94042535 = null)) THEN null ELSE cast(leverage#94042535 as float) END AS leverage#94042734, CASE WHEN ((volatility#94042536 = NA) OR (volatility#94042536 = null)) THEN null ELSE cast(volatility#94042536 as float) END AS volatility#94042736, CASE WHEN ((momentum#94042537 = NA) OR (momentum#94042537 = null)) THEN null ELSE cast(momentum#94042537 as float) END AS momentum#94042738, CASE WHEN ((yield#94042538 = NA) OR (yield#94042538 = null)) THEN null ELSE cast(yield#94042538 as float) END AS yield#94042740, CASE WHEN ((numcos#94042539 = NA) OR (numcos#94042539 = null)) THEN null ELSE cast(numcos#94042539 as float) END AS numcos#94042742, CASE WHEN ((numdates#94042540 = NA) OR (numdates#94042540 = null)) THEN null ELSE cast(numdates#94042540 as float) END AS numdates#94042745, CASE WHEN ((annual_bmret#94042541 = NA) OR (annual_bmret#94042541 = null)) THEN null ELSE cast(annual_bmret#94042541 as float) END AS annual_bmret#94042747, CASE WHEN ((annual_ret#94042542 = NA) OR (annual_ret#94042542 = null)) THEN null ELSE cast(annual_ret#94042542 as float) END AS annual_ret#94042748, CASE WHEN ((std_ret#94042543 = NA) OR (std_ret#94042543 = null)) THEN null ELSE cast(std_ret#94042543 as float) END AS std_ret#94042750, CASE WHEN ((Sharpe_ret#94042544 = NA) OR (Sharpe_ret#94042544 = null)) THEN null ELSE cast(Sharpe_ret#94042544 as float) END AS Sharpe_ret#94042751, CASE WHEN ((PctPos_ret#94042545 = NA) OR (PctPos_ret#94042545 = null)) THEN null ELSE cast(PctPos_ret#94042545 as float) END AS PctPos_ret#94042752, CASE WHEN ((TR_ret#94042546 = NA) OR (TR_ret#94042546 = null)) THEN null ELSE cast(TR_ret#94042546 as float) END AS TR_ret#94042753, CASE WHEN ((IR_ret#94042547 = NA) OR (IR_ret#94042547 = null)) THEN null ELSE cast(IR_ret#94042547 as float) END AS IR_ret#94042754, CASE WHEN ((annual_resret#94042548 = NA) OR (annual_resret#94042548 = null)) THEN null ELSE cast(annual_resret#94042548 as float) END AS annual_resret#94042755, CASE WHEN ((std_resret#94042549 = NA) OR (std_resret#94042549 = null)) THEN null ELSE cast(std_resret#94042549 as float) END AS std_resret#94042756, CASE WHEN ((Sharpe_resret#94042550 = NA) OR (Sharpe_resret#94042550 = null)) THEN null ELSE cast(Sharpe_resret#94042550 as float) END AS Sharpe_resret#94042757, CASE WHEN ((PctPos_resret#94042551 = NA) OR (PctPos_resret#94042551 = null)) THEN null ELSE cast(PctPos_resret#94042551 as float) END AS PctPos_resret#94042758, ... 9 more fields]
+- FileScan csv [overall#94042528,cap#94042529,retIC#94042530,resretIC#94042531,size#94042532,value#94042533,growth#94042534,leverage#94042535,volatility#94042536,momentum#94042537,yield#94042538,numcos#94042539,numdates#94042540,annual_bmret#94042541,annual_ret#94042542,std_ret#94042543,Sharpe_ret#94042544,PctPos_ret#94042545,TR_ret#94042546,IR_ret#94042547,annual_resret#94042548,std_resret#94042549,Sharpe_resret#94042550,PctPos_resret#94042551,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(3) InMemoryTableScan
Output [8]: [cap#94042720, growth#94042731, leverage#94042734, momentum#94042738, size#94042727, value#94042729, volatility#94042736, yield#94042740]
Arguments: [cap#94042720, growth#94042731, leverage#94042734, momentum#94042738, size#94042727, value#94042729, volatility#94042736, yield#94042740], [isnotnull(cap#94042720), (cap#94042720 = 0.0)]
(4) InMemoryRelation
Arguments: [overall#94042718, cap#94042720, retIC#94042722, resretIC#94042725, size#94042727, value#94042729, growth#94042731, leverage#94042734, volatility#94042736, momentum#94042738, yield#94042740, numcos#94042742, numdates#94042745, annual_bmret#94042747, annual_ret#94042748, std_ret#94042750, Sharpe_ret#94042751, PctPos_ret#94042752, TR_ret#94042753, IR_ret#94042754, annual_resret#94042755, std_resret#94042756, Sharpe_resret#94042757, PctPos_resret#94042758, ... 9 more fields], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((overall#94042528 = NA) OR (overall#94042528 = null)) THEN null ELSE cast(overall#94042528 as int) END AS overall#94042718, CASE WHEN ((cap#94042529 = NA) OR (cap#94042529 = null)) THEN null ELSE cast(cap#94042529 as float) END AS cap#94042720, CASE WHEN ((retIC#94042530 = NA) OR (retIC#94042530 = null)) THEN null ELSE cast(retIC#94042530 as float) END AS retIC#94042722, CASE WHEN ((resretIC#94042531 = NA) OR (resretIC#94042531 = null)) THEN null ELSE cast(resretIC#94042531 as float) END AS resretIC#94042725, CASE WHEN ((size#94042532 = NA) OR (size#94042532 = null)) THEN null ELSE cast(size#94042532 as float) END AS size#94042727, CASE WHEN ((value#94042533 = NA) OR (value#94042533 = null)) THEN null ELSE cast(value#94042533 as float) END AS value#94042729, CASE WHEN ((growth#94042534 = NA) OR (growth#94042534 = null)) THEN null ELSE cast(growth#94042534 as float) END AS growth#94042731, CASE WHEN ((leverage#94042535 = NA) OR (leverage#94042535 = null)) THEN null ELSE cast(leverage#94042535 as float) END AS leverage#94042734, CASE WHEN ((volatility#94042536 = NA) OR (volatility#94042536 = null)) THEN null ELSE cast(volatility#94042536 as float) END AS volatility#94042736, CASE WHEN ((momentum#94042537 = NA) OR (momentum#94042537 = null)) THEN null ELSE cast(momentum#94042537 as float) END AS momentum#94042738, CASE WHEN ((yield#94042538 = NA) OR (yield#94042538 = null)) THEN null ELSE cast(yield#94042538 as float) END AS yield#94042740, CASE WHEN ((numcos#94042539 = NA) OR (numcos#94042539 = null)) THEN null ELSE cast(numcos#94042539 as float) END AS numcos#94042742, CASE WHEN ((numdates#94042540 = NA) OR (numdates#94042540 = null)) THEN null ELSE cast(numdates#94042540 as float) END AS numdates#94042745, CASE WHEN ((annual_bmret#94042541 = NA) OR (annual_bmret#94042541 = null)) THEN null ELSE cast(annual_bmret#94042541 as float) END AS annual_bmret#94042747, CASE WHEN ((annual_ret#94042542 = NA) OR (annual_ret#94042542 = null)) THEN null ELSE cast(annual_ret#94042542 as float) END AS annual_ret#94042748, CASE WHEN ((std_ret#94042543 = NA) OR (std_ret#94042543 = null)) THEN null ELSE cast(std_ret#94042543 as float) END AS std_ret#94042750, CASE WHEN ((Sharpe_ret#94042544 = NA) OR (Sharpe_ret#94042544 = null)) THEN null ELSE cast(Sharpe_ret#94042544 as float) END AS Sharpe_ret#94042751, CASE WHEN ((PctPos_ret#94042545 = NA) OR (PctPos_ret#94042545 = null)) THEN null ELSE cast(PctPos_ret#94042545 as float) END AS PctPos_ret#94042752, CASE WHEN ((TR_ret#94042546 = NA) OR (TR_ret#94042546 = null)) THEN null ELSE cast(TR_ret#94042546 as float) END AS TR_ret#94042753, CASE WHEN ((IR_ret#94042547 = NA) OR (IR_ret#94042547 = null)) THEN null ELSE cast(IR_ret#94042547 as float) END AS IR_ret#94042754, CASE WHEN ((annual_resret#94042548 = NA) OR (annual_resret#94042548 = null)) THEN null ELSE cast(annual_resret#94042548 as float) END AS annual_resret#94042755, CASE WHEN ((std_resret#94042549 = NA) OR (std_resret#94042549 = null)) THEN null ELSE cast(std_resret#94042549 as float) END AS std_resret#94042756, CASE WHEN ((Sharpe_resret#94042550 = NA) OR (Sharpe_resret#94042550 = null)) THEN null ELSE cast(Sharpe_resret#94042550 as float) END AS Sharpe_resret#94042757, CASE WHEN ((PctPos_resret#94042551 = NA) OR (PctPos_resret#94042551 = null)) THEN null ELSE cast(PctPos_resret#94042551 as float) END AS PctPos_resret#94042758, ... 9 more fields]
+- FileScan csv [overall#94042528,cap#94042529,retIC#94042530,resretIC#94042531,size#94042532,value#94042533,growth#94042534,leverage#94042535,volatility#94042536,momentum#94042537,yield#94042538,numcos#94042539,numdates#94042540,annual_bmret#94042541,annual_ret#94042542,std_ret#94042543,Sharpe_ret#94042544,PctPos_ret#94042545,TR_ret#94042546,IR_ret#94042547,annual_resret#94042548,std_resret#94042549,Sharpe_resret#94042550,PctPos_resret#94042551,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(5) Scan csv
Output [33]: [overall#94042528, cap#94042529, retIC#94042530, resretIC#94042531, size#94042532, value#94042533, growth#94042534, leverage#94042535, volatility#94042536, momentum#94042537, yield#94042538, numcos#94042539, numdates#94042540, annual_bmret#94042541, annual_ret#94042542, std_ret#94042543, Sharpe_ret#94042544, PctPos_ret#94042545, TR_ret#94042546, IR_ret#94042547, annual_resret#94042548, std_resret#94042549, Sharpe_resret#94042550, PctPos_resret#94042551, TR_resret#94042552, IR_resret#94042553, annual_retnet#94042554, std_retnet#94042555, Sharpe_retnet#94042556, PctPos_retnet#94042557, TR_retnet#94042558, IR_retnet#94042559, turnover#94042560]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/stats_overall.csv]
ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:string,leverage:string,volatility:string,momentum:string,yield:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string>
(6) Project [codegen id : 1]
Output [33]: [CASE WHEN ((overall#94042528 = NA) OR (overall#94042528 = null)) THEN null ELSE cast(overall#94042528 as int) END AS overall#94042718, CASE WHEN ((cap#94042529 = NA) OR (cap#94042529 = null)) THEN null ELSE cast(cap#94042529 as float) END AS cap#94042720, CASE WHEN ((retIC#94042530 = NA) OR (retIC#94042530 = null)) THEN null ELSE cast(retIC#94042530 as float) END AS retIC#94042722, CASE WHEN ((resretIC#94042531 = NA) OR (resretIC#94042531 = null)) THEN null ELSE cast(resretIC#94042531 as float) END AS resretIC#94042725, CASE WHEN ((size#94042532 = NA) OR (size#94042532 = null)) THEN null ELSE cast(size#94042532 as float) END AS size#94042727, CASE WHEN ((value#94042533 = NA) OR (value#94042533 = null)) THEN null ELSE cast(value#94042533 as float) END AS value#94042729, CASE WHEN ((growth#94042534 = NA) OR (growth#94042534 = null)) THEN null ELSE cast(growth#94042534 as float) END AS growth#94042731, CASE WHEN ((leverage#94042535 = NA) OR (leverage#94042535 = null)) THEN null ELSE cast(leverage#94042535 as float) END AS leverage#94042734, CASE WHEN ((volatility#94042536 = NA) OR (volatility#94042536 = null)) THEN null ELSE cast(volatility#94042536 as float) END AS volatility#94042736, CASE WHEN ((momentum#94042537 = NA) OR (momentum#94042537 = null)) THEN null ELSE cast(momentum#94042537 as float) END AS momentum#94042738, CASE WHEN ((yield#94042538 = NA) OR (yield#94042538 = null)) THEN null ELSE cast(yield#94042538 as float) END AS yield#94042740, CASE WHEN ((numcos#94042539 = NA) OR (numcos#94042539 = null)) THEN null ELSE cast(numcos#94042539 as float) END AS numcos#94042742, CASE WHEN ((numdates#94042540 = NA) OR (numdates#94042540 = null)) THEN null ELSE cast(numdates#94042540 as float) END AS numdates#94042745, CASE WHEN ((annual_bmret#94042541 = NA) OR (annual_bmret#94042541 = null)) THEN null ELSE cast(annual_bmret#94042541 as float) END AS annual_bmret#94042747, CASE WHEN ((annual_ret#94042542 = NA) OR (annual_ret#94042542 = null)) THEN null ELSE cast(annual_ret#94042542 as float) END AS annual_ret#94042748, CASE WHEN ((std_ret#94042543 = NA) OR (std_ret#94042543 = null)) THEN null ELSE cast(std_ret#94042543 as float) END AS std_ret#94042750, CASE WHEN ((Sharpe_ret#94042544 = NA) OR (Sharpe_ret#94042544 = null)) THEN null ELSE cast(Sharpe_ret#94042544 as float) END AS Sharpe_ret#94042751, CASE WHEN ((PctPos_ret#94042545 = NA) OR (PctPos_ret#94042545 = null)) THEN null ELSE cast(PctPos_ret#94042545 as float) END AS PctPos_ret#94042752, CASE WHEN ((TR_ret#94042546 = NA) OR (TR_ret#94042546 = null)) THEN null ELSE cast(TR_ret#94042546 as float) END AS TR_ret#94042753, CASE WHEN ((IR_ret#94042547 = NA) OR (IR_ret#94042547 = null)) THEN null ELSE cast(IR_ret#94042547 as float) END AS IR_ret#94042754, CASE WHEN ((annual_resret#94042548 = NA) OR (annual_resret#94042548 = null)) THEN null ELSE cast(annual_resret#94042548 as float) END AS annual_resret#94042755, CASE WHEN ((std_resret#94042549 = NA) OR (std_resret#94042549 = null)) THEN null ELSE cast(std_resret#94042549 as float) END AS std_resret#94042756, CASE WHEN ((Sharpe_resret#94042550 = NA) OR (Sharpe_resret#94042550 = null)) THEN null ELSE cast(Sharpe_resret#94042550 as float) END AS Sharpe_resret#94042757, CASE WHEN ((PctPos_resret#94042551 = NA) OR (PctPos_resret#94042551 = null)) THEN null ELSE cast(PctPos_resret#94042551 as float) END AS PctPos_resret#94042758, CASE WHEN ((TR_resret#94042552 = NA) OR (TR_resret#94042552 = null)) THEN null ELSE cast(TR_resret#94042552 as float) END AS TR_resret#94042759, CASE WHEN ((IR_resret#94042553 = NA) OR (IR_resret#94042553 = null)) THEN null ELSE cast(IR_resret#94042553 as float) END AS IR_resret#94042760, CASE WHEN ((annual_retnet#94042554 = NA) OR (annual_retnet#94042554 = null)) THEN null ELSE cast(annual_retnet#94042554 as float) END AS annual_retnet#94042761, CASE WHEN ((std_retnet#94042555 = NA) OR (std_retnet#94042555 = null)) THEN null ELSE cast(std_retnet#94042555 as float) END AS std_retnet#94042762, CASE WHEN ((Sharpe_retnet#94042556 = NA) OR (Sharpe_retnet#94042556 = null)) THEN null ELSE cast(Sharpe_retnet#94042556 as float) END AS Sharpe_retnet#94042788, CASE WHEN ((PctPos_retnet#94042557 = NA) OR (PctPos_retnet#94042557 = null)) THEN null ELSE cast(PctPos_retnet#94042557 as float) END AS PctPos_retnet#94042789, CASE WHEN ((TR_retnet#94042558 = NA) OR (TR_retnet#94042558 = null)) THEN null ELSE cast(TR_retnet#94042558 as float) END AS TR_retnet#94042790, CASE WHEN ((IR_retnet#94042559 = NA) OR (IR_retnet#94042559 = null)) THEN null ELSE cast(IR_retnet#94042559 as float) END AS IR_retnet#94042791, CASE WHEN ((turnover#94042560 = NA) OR (turnover#94042560 = null)) THEN null ELSE cast(turnover#94042560 as float) END AS turnover#94042793]
Input [33]: [overall#94042528, cap#94042529, retIC#94042530, resretIC#94042531, size#94042532, value#94042533, growth#94042534, leverage#94042535, volatility#94042536, momentum#94042537, yield#94042538, numcos#94042539, numdates#94042540, annual_bmret#94042541, annual_ret#94042542, std_ret#94042543, Sharpe_ret#94042544, PctPos_ret#94042545, TR_ret#94042546, IR_ret#94042547, annual_resret#94042548, std_resret#94042549, Sharpe_resret#94042550, PctPos_resret#94042551, TR_resret#94042552, IR_resret#94042553, annual_retnet#94042554, std_retnet#94042555, Sharpe_retnet#94042556, PctPos_retnet#94042557, TR_retnet#94042558, IR_retnet#94042559, turnover#94042560]
(7) ColumnarToRow [codegen id : 1]
Input [8]: [cap#94042720, growth#94042731, leverage#94042734, momentum#94042738, size#94042727, value#94042729, volatility#94042736, yield#94042740]
(8) Filter [codegen id : 1]
Input [8]: [cap#94042720, growth#94042731, leverage#94042734, momentum#94042738, size#94042727, value#94042729, volatility#94042736, yield#94042740]
Condition : (isnotnull(cap#94042720) AND (cap#94042720 = 0.0))
(9) Project [codegen id : 1]
Output [7]: [yield#94042740, volatility#94042736, momentum#94042738, size#94042727, value#94042729, growth#94042731, leverage#94042734]
Input [8]: [cap#94042720, growth#94042731, leverage#94042734, momentum#94042738, size#94042727, value#94042729, volatility#94042736, yield#94042740]
(10) ColumnarToRow [codegen id : 1]
Input [7]: [yield#94042740, volatility#94042736, momentum#94042738, size#94042727, value#94042729, growth#94042731, leverage#94042734]
(11) CollectLimit
Input [7]: [yield#94042740, volatility#94042736, momentum#94042738, size#94042727, value#94042729, growth#94042731, leverage#94042734]
Arguments: 1000000