== Physical Plan == CollectLimit (13) +- * ColumnarToRow (12) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (11) +- Exchange (10) +- * Project (9) +- * Filter (8) +- * ColumnarToRow (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [year#93984728, turnover#93984835, days_hold#93984877] Arguments: [year#93984728, turnover#93984835, days_hold#93984877] (2) InMemoryRelation Arguments: [year#93984728, turnover#93984835, days_hold#93984877], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [year#93984728 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#93984728 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7503987] +- *(1) Project [year#93984728, turnover#93984835, (1.0 / cast(turnover#93984835 as double)) AS days_hold#93984877] +- *(1) Filter isnotnull(turnover#93984835) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#93984835, year#93984728], [isnotnull(turnover#93984835)] +- InMemoryRelation [year#93984728, retIC#93984762, resretIC#93984763, numcos#93984764, numdates#93984765, annual_bmret#93984799, annual_ret#93984800, std_ret#93984801, Sharpe_ret#93984802, PctPos_ret#93984803, TR_ret#93984805, IR_ret#93984807, annual_resret#93984809, std_resret#93984811, Sharpe_resret#93984814, PctPos_resret#93984816, TR_resret#93984818, IR_resret#93984820, annual_retnet#93984823, std_retnet#93984825, Sharpe_retnet#93984827, PctPos_retnet#93984829, TR_retnet#93984831, IR_retnet#93984833, turnover#93984835], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN ((year#93984561 = NA) OR (year#93984561 = null)) THEN null ELSE cast(year#93984561 as float) END AS year#93984728, CASE WHEN ((retIC#93984562 = NA) OR (retIC#93984562 = null)) THEN null ELSE cast(retIC#93984562 as float) END AS retIC#93984762, CASE WHEN ((resretIC#93984563 = NA) OR (resretIC#93984563 = null)) THEN null ELSE cast(resretIC#93984563 as float) END AS resretIC#93984763, CASE WHEN ((numcos#93984564 = NA) OR (numcos#93984564 = null)) THEN null ELSE cast(numcos#93984564 as float) END AS numcos#93984764, CASE WHEN ((numdates#93984565 = NA) OR (numdates#93984565 = null)) THEN null ELSE cast(numdates#93984565 as int) END AS numdates#93984765, CASE WHEN ((annual_bmret#93984566 = NA) OR (annual_bmret#93984566 = null)) THEN null ELSE cast(annual_bmret#93984566 as float) END AS annual_bmret#93984799, CASE WHEN ((annual_ret#93984567 = NA) OR (annual_ret#93984567 = null)) THEN null ELSE cast(annual_ret#93984567 as float) END AS annual_ret#93984800, CASE WHEN ((std_ret#93984568 = NA) OR (std_ret#93984568 = null)) THEN null ELSE cast(std_ret#93984568 as float) END AS std_ret#93984801, CASE WHEN ((Sharpe_ret#93984569 = NA) OR (Sharpe_ret#93984569 = null)) THEN null ELSE cast(Sharpe_ret#93984569 as float) END AS Sharpe_ret#93984802, CASE WHEN ((PctPos_ret#93984570 = NA) OR (PctPos_ret#93984570 = null)) THEN null ELSE cast(PctPos_ret#93984570 as float) END AS PctPos_ret#93984803, CASE WHEN ((TR_ret#93984571 = NA) OR (TR_ret#93984571 = null)) THEN null ELSE cast(TR_ret#93984571 as float) END AS TR_ret#93984805, CASE WHEN ((IR_ret#93984572 = NA) OR (IR_ret#93984572 = null)) THEN null ELSE cast(IR_ret#93984572 as float) END AS IR_ret#93984807, CASE WHEN ((annual_resret#93984573 = NA) OR (annual_resret#93984573 = null)) THEN null ELSE cast(annual_resret#93984573 as float) END AS annual_resret#93984809, CASE WHEN ((std_resret#93984574 = NA) OR (std_resret#93984574 = null)) THEN null ELSE cast(std_resret#93984574 as float) END AS std_resret#93984811, CASE WHEN ((Sharpe_resret#93984575 = NA) OR (Sharpe_resret#93984575 = null)) THEN null ELSE cast(Sharpe_resret#93984575 as float) END AS Sharpe_resret#93984814, CASE WHEN ((PctPos_resret#93984576 = NA) OR (PctPos_resret#93984576 = null)) THEN null ELSE cast(PctPos_resret#93984576 as float) END AS PctPos_resret#93984816, CASE WHEN ((TR_resret#93984577 = NA) OR (TR_resret#93984577 = null)) THEN null ELSE cast(TR_resret#93984577 as float) END AS TR_resret#93984818, CASE WHEN ((IR_resret#93984578 = NA) OR (IR_resret#93984578 = null)) THEN null ELSE cast(IR_resret#93984578 as float) END AS IR_resret#93984820, CASE WHEN ((annual_retnet#93984579 = NA) OR (annual_retnet#93984579 = null)) THEN null ELSE cast(annual_retnet#93984579 as float) END AS annual_retnet#93984823, CASE WHEN ((std_retnet#93984580 = NA) OR (std_retnet#93984580 = null)) THEN null ELSE cast(std_retnet#93984580 as float) END AS std_retnet#93984825, CASE WHEN ((Sharpe_retnet#93984581 = NA) OR (Sharpe_retnet#93984581 = null)) THEN null ELSE cast(Sharpe_retnet#93984581 as float) END AS Sharpe_retnet#93984827, CASE WHEN ((PctPos_retnet#93984582 = NA) OR (PctPos_retnet#93984582 = null)) THEN null ELSE cast(PctPos_retnet#93984582 as float) END AS PctPos_retnet#93984829, CASE WHEN ((TR_retnet#93984583 = NA) OR (TR_retnet#93984583 = null)) THEN null ELSE cast(TR_retnet#93984583 as float) END AS TR_retnet#93984831, CASE WHEN ((IR_retnet#93984584 = NA) OR (IR_retnet#93984584 = null)) THEN null ELSE cast(IR_retnet#93984584 as float) END AS IR_retnet#93984833, CASE WHEN ((turnover#93984585 = NA) OR (turnover#93984585 = null)) THEN null ELSE cast(turnover#93984585 as float) END AS turnover#93984835] +- FileScan csv [year#93984561,retIC#93984562,resretIC#93984563,numcos#93984564,numdates#93984565,annual_bmret#93984566,annual_ret#93984567,std_ret#93984568,Sharpe_ret#93984569,PctPos_ret#93984570,TR_ret#93984571,IR_ret#93984572,annual_resret#93984573,std_resret#93984574,Sharpe_resret#93984575,PctPos_resret#93984576,TR_resret#93984577,IR_resret#93984578,annual_retnet#93984579,std_retnet#93984580,Sharpe_retnet#93984581,PctPos_retnet#93984582,TR_retnet#93984583,IR_retnet#93984584,turnover#93984585] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None), [year#93984728 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [turnover#93984835, year#93984728] Arguments: [turnover#93984835, year#93984728], [isnotnull(turnover#93984835)] (4) InMemoryRelation Arguments: [year#93984728, retIC#93984762, resretIC#93984763, numcos#93984764, numdates#93984765, annual_bmret#93984799, annual_ret#93984800, std_ret#93984801, Sharpe_ret#93984802, PctPos_ret#93984803, TR_ret#93984805, IR_ret#93984807, annual_resret#93984809, std_resret#93984811, Sharpe_resret#93984814, PctPos_resret#93984816, TR_resret#93984818, IR_resret#93984820, annual_retnet#93984823, std_retnet#93984825, Sharpe_retnet#93984827, PctPos_retnet#93984829, TR_retnet#93984831, IR_retnet#93984833, turnover#93984835], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((year#93984561 = NA) OR (year#93984561 = null)) THEN null ELSE cast(year#93984561 as float) END AS year#93984728, CASE WHEN ((retIC#93984562 = NA) OR (retIC#93984562 = null)) THEN null ELSE cast(retIC#93984562 as float) END AS retIC#93984762, CASE WHEN ((resretIC#93984563 = NA) OR (resretIC#93984563 = null)) THEN null ELSE cast(resretIC#93984563 as float) END AS resretIC#93984763, CASE WHEN ((numcos#93984564 = NA) OR (numcos#93984564 = null)) THEN null ELSE cast(numcos#93984564 as float) END AS numcos#93984764, CASE WHEN ((numdates#93984565 = NA) OR (numdates#93984565 = null)) THEN null ELSE cast(numdates#93984565 as int) END AS numdates#93984765, CASE WHEN ((annual_bmret#93984566 = NA) OR (annual_bmret#93984566 = null)) THEN null ELSE cast(annual_bmret#93984566 as float) END AS annual_bmret#93984799, CASE WHEN ((annual_ret#93984567 = NA) OR (annual_ret#93984567 = null)) THEN null ELSE cast(annual_ret#93984567 as float) END AS annual_ret#93984800, CASE WHEN ((std_ret#93984568 = NA) OR (std_ret#93984568 = null)) THEN null ELSE cast(std_ret#93984568 as float) END AS std_ret#93984801, CASE WHEN ((Sharpe_ret#93984569 = NA) OR (Sharpe_ret#93984569 = null)) THEN null ELSE cast(Sharpe_ret#93984569 as float) END AS Sharpe_ret#93984802, CASE WHEN ((PctPos_ret#93984570 = NA) OR (PctPos_ret#93984570 = null)) THEN null ELSE cast(PctPos_ret#93984570 as float) END AS PctPos_ret#93984803, CASE WHEN ((TR_ret#93984571 = NA) OR (TR_ret#93984571 = null)) THEN null ELSE cast(TR_ret#93984571 as float) END AS TR_ret#93984805, CASE WHEN ((IR_ret#93984572 = NA) OR (IR_ret#93984572 = null)) THEN null ELSE cast(IR_ret#93984572 as float) END AS IR_ret#93984807, CASE WHEN ((annual_resret#93984573 = NA) OR (annual_resret#93984573 = null)) THEN null ELSE cast(annual_resret#93984573 as float) END AS annual_resret#93984809, CASE WHEN ((std_resret#93984574 = NA) OR (std_resret#93984574 = null)) THEN null ELSE cast(std_resret#93984574 as float) END AS std_resret#93984811, CASE WHEN ((Sharpe_resret#93984575 = NA) OR (Sharpe_resret#93984575 = null)) THEN null ELSE cast(Sharpe_resret#93984575 as float) END AS Sharpe_resret#93984814, CASE WHEN ((PctPos_resret#93984576 = NA) OR (PctPos_resret#93984576 = null)) THEN null ELSE cast(PctPos_resret#93984576 as float) END AS PctPos_resret#93984816, CASE WHEN ((TR_resret#93984577 = NA) OR (TR_resret#93984577 = null)) THEN null ELSE cast(TR_resret#93984577 as float) END AS TR_resret#93984818, CASE WHEN ((IR_resret#93984578 = NA) OR (IR_resret#93984578 = null)) THEN null ELSE cast(IR_resret#93984578 as float) END AS IR_resret#93984820, CASE WHEN ((annual_retnet#93984579 = NA) OR (annual_retnet#93984579 = null)) THEN null ELSE cast(annual_retnet#93984579 as float) END AS annual_retnet#93984823, CASE WHEN ((std_retnet#93984580 = NA) OR (std_retnet#93984580 = null)) THEN null ELSE cast(std_retnet#93984580 as float) END AS std_retnet#93984825, CASE WHEN ((Sharpe_retnet#93984581 = NA) OR (Sharpe_retnet#93984581 = null)) THEN null ELSE cast(Sharpe_retnet#93984581 as float) END AS Sharpe_retnet#93984827, CASE WHEN ((PctPos_retnet#93984582 = NA) OR (PctPos_retnet#93984582 = null)) THEN null ELSE cast(PctPos_retnet#93984582 as float) END AS PctPos_retnet#93984829, CASE WHEN ((TR_retnet#93984583 = NA) OR (TR_retnet#93984583 = null)) THEN null ELSE cast(TR_retnet#93984583 as float) END AS TR_retnet#93984831, CASE WHEN ((IR_retnet#93984584 = NA) OR (IR_retnet#93984584 = null)) THEN null ELSE cast(IR_retnet#93984584 as float) END AS IR_retnet#93984833, CASE WHEN ((turnover#93984585 = NA) OR (turnover#93984585 = null)) THEN null ELSE cast(turnover#93984585 as float) END AS turnover#93984835] +- FileScan csv [year#93984561,retIC#93984562,resretIC#93984563,numcos#93984564,numdates#93984565,annual_bmret#93984566,annual_ret#93984567,std_ret#93984568,Sharpe_ret#93984569,PctPos_ret#93984570,TR_ret#93984571,IR_ret#93984572,annual_resret#93984573,std_resret#93984574,Sharpe_resret#93984575,PctPos_resret#93984576,TR_resret#93984577,IR_resret#93984578,annual_retnet#93984579,std_retnet#93984580,Sharpe_retnet#93984581,PctPos_retnet#93984582,TR_retnet#93984583,IR_retnet#93984584,turnover#93984585] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None) (5) Scan csv Output [25]: [year#93984561, retIC#93984562, resretIC#93984563, numcos#93984564, numdates#93984565, annual_bmret#93984566, annual_ret#93984567, std_ret#93984568, Sharpe_ret#93984569, PctPos_ret#93984570, TR_ret#93984571, IR_ret#93984572, annual_resret#93984573, std_resret#93984574, Sharpe_resret#93984575, PctPos_resret#93984576, TR_resret#93984577, IR_resret#93984578, annual_retnet#93984579, std_retnet#93984580, Sharpe_retnet#93984581, PctPos_retnet#93984582, TR_retnet#93984583, IR_retnet#93984584, turnover#93984585] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/estimize_signal_history/estimizesignal_postearnings/stats_year.csv] ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string> (6) Project [codegen id : 1] Output [25]: [CASE WHEN ((year#93984561 = NA) OR (year#93984561 = null)) THEN null ELSE cast(year#93984561 as float) END AS year#93984728, CASE WHEN ((retIC#93984562 = NA) OR (retIC#93984562 = null)) THEN null ELSE cast(retIC#93984562 as float) END AS retIC#93984762, CASE WHEN ((resretIC#93984563 = NA) OR (resretIC#93984563 = null)) THEN null ELSE cast(resretIC#93984563 as float) END AS resretIC#93984763, CASE WHEN ((numcos#93984564 = NA) OR (numcos#93984564 = null)) THEN null ELSE cast(numcos#93984564 as float) END AS numcos#93984764, CASE WHEN ((numdates#93984565 = NA) OR (numdates#93984565 = null)) THEN null ELSE cast(numdates#93984565 as int) END AS numdates#93984765, CASE WHEN ((annual_bmret#93984566 = NA) OR (annual_bmret#93984566 = null)) THEN null ELSE cast(annual_bmret#93984566 as float) END AS annual_bmret#93984799, CASE WHEN ((annual_ret#93984567 = NA) OR (annual_ret#93984567 = null)) THEN null ELSE cast(annual_ret#93984567 as float) END AS annual_ret#93984800, CASE WHEN ((std_ret#93984568 = NA) OR (std_ret#93984568 = null)) THEN null ELSE cast(std_ret#93984568 as float) END AS std_ret#93984801, CASE WHEN ((Sharpe_ret#93984569 = NA) OR (Sharpe_ret#93984569 = null)) THEN null ELSE cast(Sharpe_ret#93984569 as float) END AS Sharpe_ret#93984802, CASE WHEN ((PctPos_ret#93984570 = NA) OR (PctPos_ret#93984570 = null)) THEN null ELSE cast(PctPos_ret#93984570 as float) END AS PctPos_ret#93984803, CASE WHEN ((TR_ret#93984571 = NA) OR (TR_ret#93984571 = null)) THEN null ELSE cast(TR_ret#93984571 as float) END AS TR_ret#93984805, CASE WHEN ((IR_ret#93984572 = NA) OR (IR_ret#93984572 = null)) THEN null ELSE cast(IR_ret#93984572 as float) END AS IR_ret#93984807, CASE WHEN ((annual_resret#93984573 = NA) OR (annual_resret#93984573 = null)) THEN null ELSE cast(annual_resret#93984573 as float) END AS annual_resret#93984809, CASE WHEN ((std_resret#93984574 = NA) OR (std_resret#93984574 = null)) THEN null ELSE cast(std_resret#93984574 as float) END AS std_resret#93984811, CASE WHEN ((Sharpe_resret#93984575 = NA) OR (Sharpe_resret#93984575 = null)) THEN null ELSE cast(Sharpe_resret#93984575 as float) END AS Sharpe_resret#93984814, CASE WHEN ((PctPos_resret#93984576 = NA) OR (PctPos_resret#93984576 = null)) THEN null ELSE cast(PctPos_resret#93984576 as float) END AS PctPos_resret#93984816, CASE WHEN ((TR_resret#93984577 = NA) OR (TR_resret#93984577 = null)) THEN null ELSE cast(TR_resret#93984577 as float) END AS TR_resret#93984818, CASE WHEN ((IR_resret#93984578 = NA) OR (IR_resret#93984578 = null)) THEN null ELSE cast(IR_resret#93984578 as float) END AS IR_resret#93984820, CASE WHEN ((annual_retnet#93984579 = NA) OR (annual_retnet#93984579 = null)) THEN null ELSE cast(annual_retnet#93984579 as float) END AS annual_retnet#93984823, CASE WHEN ((std_retnet#93984580 = NA) OR (std_retnet#93984580 = null)) THEN null ELSE cast(std_retnet#93984580 as float) END AS std_retnet#93984825, CASE WHEN ((Sharpe_retnet#93984581 = NA) OR (Sharpe_retnet#93984581 = null)) THEN null ELSE cast(Sharpe_retnet#93984581 as float) END AS Sharpe_retnet#93984827, CASE WHEN ((PctPos_retnet#93984582 = NA) OR (PctPos_retnet#93984582 = null)) THEN null ELSE cast(PctPos_retnet#93984582 as float) END AS PctPos_retnet#93984829, CASE WHEN ((TR_retnet#93984583 = NA) OR (TR_retnet#93984583 = null)) THEN null ELSE cast(TR_retnet#93984583 as float) END AS TR_retnet#93984831, CASE WHEN ((IR_retnet#93984584 = NA) OR (IR_retnet#93984584 = null)) THEN null ELSE cast(IR_retnet#93984584 as float) END AS IR_retnet#93984833, CASE WHEN ((turnover#93984585 = NA) OR (turnover#93984585 = null)) THEN null ELSE cast(turnover#93984585 as float) END AS turnover#93984835] Input [25]: [year#93984561, retIC#93984562, resretIC#93984563, numcos#93984564, numdates#93984565, annual_bmret#93984566, annual_ret#93984567, std_ret#93984568, Sharpe_ret#93984569, PctPos_ret#93984570, TR_ret#93984571, IR_ret#93984572, annual_resret#93984573, std_resret#93984574, Sharpe_resret#93984575, PctPos_resret#93984576, TR_resret#93984577, IR_resret#93984578, annual_retnet#93984579, std_retnet#93984580, Sharpe_retnet#93984581, PctPos_retnet#93984582, TR_retnet#93984583, IR_retnet#93984584, turnover#93984585] (7) ColumnarToRow [codegen id : 1] Input [2]: [turnover#93984835, year#93984728] (8) Filter [codegen id : 1] Input [2]: [turnover#93984835, year#93984728] Condition : isnotnull(turnover#93984835) (9) Project [codegen id : 1] Output [3]: [year#93984728, turnover#93984835, (1.0 / cast(turnover#93984835 as double)) AS days_hold#93984877] Input [2]: [turnover#93984835, year#93984728] (10) Exchange Input [3]: [year#93984728, turnover#93984835, days_hold#93984877] Arguments: rangepartitioning(year#93984728 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7503987] (11) Sort [codegen id : 2] Input [3]: [year#93984728, turnover#93984835, days_hold#93984877] Arguments: [year#93984728 ASC NULLS FIRST], true, 0 (12) ColumnarToRow [codegen id : 1] Input [3]: [year#93984728, turnover#93984835, days_hold#93984877] (13) CollectLimit Input [3]: [year#93984728, turnover#93984835, days_hold#93984877] Arguments: 10000