== Physical Plan == CollectLimit (13) +- * ColumnarToRow (12) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (11) +- Exchange (10) +- * Project (9) +- * Filter (8) +- * ColumnarToRow (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [year#94243861, turnover#94244018, days_hold#94244065] Arguments: [year#94243861, turnover#94244018, days_hold#94244065] (2) InMemoryRelation Arguments: [year#94243861, turnover#94244018, days_hold#94244065], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [year#94243861 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94243861 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7524462] +- *(1) Project [year#94243861, turnover#94244018, (1.0 / cast(turnover#94244018 as double)) AS days_hold#94244065] +- *(1) Filter isnotnull(turnover#94244018) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94244018, year#94243861], [isnotnull(turnover#94244018)] +- InMemoryRelation [year#94243861, retIC#94243862, resretIC#94243863, numcos#94243865, numdates#94243866, annual_bmret#94243867, annual_ret#94243868, std_ret#94243869, Sharpe_ret#94243870, PctPos_ret#94243871, TR_ret#94243872, IR_ret#94243873, annual_resret#94243907, std_resret#94243908, Sharpe_resret#94243909, PctPos_resret#94243943, TR_resret#94243944, IR_resret#94243945, annual_retnet#94243946, std_retnet#94243980, Sharpe_retnet#94243981, PctPos_retnet#94243982, TR_retnet#94243983, IR_retnet#94243984, turnover#94244018], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN ((year#94243760 = NA) OR (year#94243760 = null)) THEN null ELSE cast(year#94243760 as float) END AS year#94243861, CASE WHEN ((retIC#94243761 = NA) OR (retIC#94243761 = null)) THEN null ELSE cast(retIC#94243761 as float) END AS retIC#94243862, CASE WHEN ((resretIC#94243762 = NA) OR (resretIC#94243762 = null)) THEN null ELSE cast(resretIC#94243762 as float) END AS resretIC#94243863, CASE WHEN ((numcos#94243763 = NA) OR (numcos#94243763 = null)) THEN null ELSE cast(numcos#94243763 as float) END AS numcos#94243865, CASE WHEN ((numdates#94243764 = NA) OR (numdates#94243764 = null)) THEN null ELSE cast(numdates#94243764 as int) END AS numdates#94243866, CASE WHEN ((annual_bmret#94243765 = NA) OR (annual_bmret#94243765 = null)) THEN null ELSE cast(annual_bmret#94243765 as float) END AS annual_bmret#94243867, CASE WHEN ((annual_ret#94243766 = NA) OR (annual_ret#94243766 = null)) THEN null ELSE cast(annual_ret#94243766 as float) END AS annual_ret#94243868, CASE WHEN ((std_ret#94243767 = NA) OR (std_ret#94243767 = null)) THEN null ELSE cast(std_ret#94243767 as float) END AS std_ret#94243869, CASE WHEN ((Sharpe_ret#94243768 = NA) OR (Sharpe_ret#94243768 = null)) THEN null ELSE cast(Sharpe_ret#94243768 as float) END AS Sharpe_ret#94243870, CASE WHEN ((PctPos_ret#94243769 = NA) OR (PctPos_ret#94243769 = null)) THEN null ELSE cast(PctPos_ret#94243769 as float) END AS PctPos_ret#94243871, CASE WHEN ((TR_ret#94243770 = NA) OR (TR_ret#94243770 = null)) THEN null ELSE cast(TR_ret#94243770 as float) END AS TR_ret#94243872, CASE WHEN ((IR_ret#94243771 = NA) OR (IR_ret#94243771 = null)) THEN null ELSE cast(IR_ret#94243771 as float) END AS IR_ret#94243873, CASE WHEN ((annual_resret#94243772 = NA) OR (annual_resret#94243772 = null)) THEN null ELSE cast(annual_resret#94243772 as float) END AS annual_resret#94243907, CASE WHEN ((std_resret#94243773 = NA) OR (std_resret#94243773 = null)) THEN null ELSE cast(std_resret#94243773 as float) END AS std_resret#94243908, CASE WHEN ((Sharpe_resret#94243774 = NA) OR (Sharpe_resret#94243774 = null)) THEN null ELSE cast(Sharpe_resret#94243774 as float) END AS Sharpe_resret#94243909, CASE WHEN ((PctPos_resret#94243775 = NA) OR (PctPos_resret#94243775 = null)) THEN null ELSE cast(PctPos_resret#94243775 as float) END AS PctPos_resret#94243943, CASE WHEN ((TR_resret#94243776 = NA) OR (TR_resret#94243776 = null)) THEN null ELSE cast(TR_resret#94243776 as float) END AS TR_resret#94243944, CASE WHEN ((IR_resret#94243777 = NA) OR (IR_resret#94243777 = null)) THEN null ELSE cast(IR_resret#94243777 as float) END AS IR_resret#94243945, CASE WHEN ((annual_retnet#94243778 = NA) OR (annual_retnet#94243778 = null)) THEN null ELSE cast(annual_retnet#94243778 as float) END AS annual_retnet#94243946, CASE WHEN ((std_retnet#94243779 = NA) OR (std_retnet#94243779 = null)) THEN null ELSE cast(std_retnet#94243779 as float) END AS std_retnet#94243980, CASE WHEN ((Sharpe_retnet#94243780 = NA) OR (Sharpe_retnet#94243780 = null)) THEN null ELSE cast(Sharpe_retnet#94243780 as float) END AS Sharpe_retnet#94243981, CASE WHEN ((PctPos_retnet#94243781 = NA) OR (PctPos_retnet#94243781 = null)) THEN null ELSE cast(PctPos_retnet#94243781 as float) END AS PctPos_retnet#94243982, CASE WHEN ((TR_retnet#94243782 = NA) OR (TR_retnet#94243782 = null)) THEN null ELSE cast(TR_retnet#94243782 as float) END AS TR_retnet#94243983, CASE WHEN ((IR_retnet#94243783 = NA) OR (IR_retnet#94243783 = null)) THEN null ELSE cast(IR_retnet#94243783 as float) END AS IR_retnet#94243984, CASE WHEN ((turnover#94243784 = NA) OR (turnover#94243784 = null)) THEN null ELSE cast(turnover#94243784 as float) END AS turnover#94244018] +- FileScan csv [year#94243760,retIC#94243761,resretIC#94243762,numcos#94243763,numdates#94243764,annual_bmret#94243765,annual_ret#94243766,std_ret#94243767,Sharpe_ret#94243768,PctPos_ret#94243769,TR_ret#94243770,IR_ret#94243771,annual_resret#94243772,std_resret#94243773,Sharpe_resret#94243774,PctPos_resret#94243775,TR_resret#94243776,IR_resret#94243777,annual_retnet#94243778,std_retnet#94243779,Sharpe_retnet#94243780,PctPos_retnet#94243781,TR_retnet#94243782,IR_retnet#94243783,turnover#94243784] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/aqis_history/weight/st..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None), [year#94243861 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [turnover#94244018, year#94243861] Arguments: [turnover#94244018, year#94243861], [isnotnull(turnover#94244018)] (4) InMemoryRelation Arguments: [year#94243861, retIC#94243862, resretIC#94243863, numcos#94243865, numdates#94243866, annual_bmret#94243867, annual_ret#94243868, std_ret#94243869, Sharpe_ret#94243870, PctPos_ret#94243871, TR_ret#94243872, IR_ret#94243873, annual_resret#94243907, std_resret#94243908, Sharpe_resret#94243909, PctPos_resret#94243943, TR_resret#94243944, IR_resret#94243945, annual_retnet#94243946, std_retnet#94243980, Sharpe_retnet#94243981, PctPos_retnet#94243982, TR_retnet#94243983, IR_retnet#94243984, turnover#94244018], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((year#94243760 = NA) OR (year#94243760 = null)) THEN null ELSE cast(year#94243760 as float) END AS year#94243861, CASE WHEN ((retIC#94243761 = NA) OR (retIC#94243761 = null)) THEN null ELSE cast(retIC#94243761 as float) END AS retIC#94243862, CASE WHEN ((resretIC#94243762 = NA) OR (resretIC#94243762 = null)) THEN null ELSE cast(resretIC#94243762 as float) END AS resretIC#94243863, CASE WHEN ((numcos#94243763 = NA) OR (numcos#94243763 = null)) THEN null ELSE cast(numcos#94243763 as float) END AS numcos#94243865, CASE WHEN ((numdates#94243764 = NA) OR (numdates#94243764 = null)) THEN null ELSE cast(numdates#94243764 as int) END AS numdates#94243866, CASE WHEN ((annual_bmret#94243765 = NA) OR (annual_bmret#94243765 = null)) THEN null ELSE cast(annual_bmret#94243765 as float) END AS annual_bmret#94243867, CASE WHEN ((annual_ret#94243766 = NA) OR (annual_ret#94243766 = null)) THEN null ELSE cast(annual_ret#94243766 as float) END AS annual_ret#94243868, CASE WHEN ((std_ret#94243767 = NA) OR (std_ret#94243767 = null)) THEN null ELSE cast(std_ret#94243767 as float) END AS std_ret#94243869, CASE WHEN ((Sharpe_ret#94243768 = NA) OR (Sharpe_ret#94243768 = null)) THEN null ELSE cast(Sharpe_ret#94243768 as float) END AS Sharpe_ret#94243870, CASE WHEN ((PctPos_ret#94243769 = NA) OR (PctPos_ret#94243769 = null)) THEN null ELSE cast(PctPos_ret#94243769 as float) END AS PctPos_ret#94243871, CASE WHEN ((TR_ret#94243770 = NA) OR (TR_ret#94243770 = null)) THEN null ELSE cast(TR_ret#94243770 as float) END AS TR_ret#94243872, CASE WHEN ((IR_ret#94243771 = NA) OR (IR_ret#94243771 = null)) THEN null ELSE cast(IR_ret#94243771 as float) END AS IR_ret#94243873, CASE WHEN ((annual_resret#94243772 = NA) OR (annual_resret#94243772 = null)) THEN null ELSE cast(annual_resret#94243772 as float) END AS annual_resret#94243907, CASE WHEN ((std_resret#94243773 = NA) OR (std_resret#94243773 = null)) THEN null ELSE cast(std_resret#94243773 as float) END AS std_resret#94243908, CASE WHEN ((Sharpe_resret#94243774 = NA) OR (Sharpe_resret#94243774 = null)) THEN null ELSE cast(Sharpe_resret#94243774 as float) END AS Sharpe_resret#94243909, CASE WHEN ((PctPos_resret#94243775 = NA) OR (PctPos_resret#94243775 = null)) THEN null ELSE cast(PctPos_resret#94243775 as float) END AS PctPos_resret#94243943, CASE WHEN ((TR_resret#94243776 = NA) OR (TR_resret#94243776 = null)) THEN null ELSE cast(TR_resret#94243776 as float) END AS TR_resret#94243944, CASE WHEN ((IR_resret#94243777 = NA) OR (IR_resret#94243777 = null)) THEN null ELSE cast(IR_resret#94243777 as float) END AS IR_resret#94243945, CASE WHEN ((annual_retnet#94243778 = NA) OR (annual_retnet#94243778 = null)) THEN null ELSE cast(annual_retnet#94243778 as float) END AS annual_retnet#94243946, CASE WHEN ((std_retnet#94243779 = NA) OR (std_retnet#94243779 = null)) THEN null ELSE cast(std_retnet#94243779 as float) END AS std_retnet#94243980, CASE WHEN ((Sharpe_retnet#94243780 = NA) OR (Sharpe_retnet#94243780 = null)) THEN null ELSE cast(Sharpe_retnet#94243780 as float) END AS Sharpe_retnet#94243981, CASE WHEN ((PctPos_retnet#94243781 = NA) OR (PctPos_retnet#94243781 = null)) THEN null ELSE cast(PctPos_retnet#94243781 as float) END AS PctPos_retnet#94243982, CASE WHEN ((TR_retnet#94243782 = NA) OR (TR_retnet#94243782 = null)) THEN null ELSE cast(TR_retnet#94243782 as float) END AS TR_retnet#94243983, CASE WHEN ((IR_retnet#94243783 = NA) OR (IR_retnet#94243783 = null)) THEN null ELSE cast(IR_retnet#94243783 as float) END AS IR_retnet#94243984, CASE WHEN ((turnover#94243784 = NA) OR (turnover#94243784 = null)) THEN null ELSE cast(turnover#94243784 as float) END AS turnover#94244018] +- FileScan csv [year#94243760,retIC#94243761,resretIC#94243762,numcos#94243763,numdates#94243764,annual_bmret#94243765,annual_ret#94243766,std_ret#94243767,Sharpe_ret#94243768,PctPos_ret#94243769,TR_ret#94243770,IR_ret#94243771,annual_resret#94243772,std_resret#94243773,Sharpe_resret#94243774,PctPos_resret#94243775,TR_resret#94243776,IR_resret#94243777,annual_retnet#94243778,std_retnet#94243779,Sharpe_retnet#94243780,PctPos_retnet#94243781,TR_retnet#94243782,IR_retnet#94243783,turnover#94243784] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/aqis_history/weight/st..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None) (5) Scan csv Output [25]: [year#94243760, retIC#94243761, resretIC#94243762, numcos#94243763, numdates#94243764, annual_bmret#94243765, annual_ret#94243766, std_ret#94243767, Sharpe_ret#94243768, PctPos_ret#94243769, TR_ret#94243770, IR_ret#94243771, annual_resret#94243772, std_resret#94243773, Sharpe_resret#94243774, PctPos_resret#94243775, TR_resret#94243776, IR_resret#94243777, annual_retnet#94243778, std_retnet#94243779, Sharpe_retnet#94243780, PctPos_retnet#94243781, TR_retnet#94243782, IR_retnet#94243783, turnover#94243784] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/aqis_history/weight/stats_year.csv] ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string> (6) Project [codegen id : 1] Output [25]: [CASE WHEN ((year#94243760 = NA) OR (year#94243760 = null)) THEN null ELSE cast(year#94243760 as float) END AS year#94243861, CASE WHEN ((retIC#94243761 = NA) OR (retIC#94243761 = null)) THEN null ELSE cast(retIC#94243761 as float) END AS retIC#94243862, CASE WHEN ((resretIC#94243762 = NA) OR (resretIC#94243762 = null)) THEN null ELSE cast(resretIC#94243762 as float) END AS resretIC#94243863, CASE WHEN ((numcos#94243763 = NA) OR (numcos#94243763 = null)) THEN null ELSE cast(numcos#94243763 as float) END AS numcos#94243865, CASE WHEN ((numdates#94243764 = NA) OR (numdates#94243764 = null)) THEN null ELSE cast(numdates#94243764 as int) END AS numdates#94243866, CASE WHEN ((annual_bmret#94243765 = NA) OR (annual_bmret#94243765 = null)) THEN null ELSE cast(annual_bmret#94243765 as float) END AS annual_bmret#94243867, CASE WHEN ((annual_ret#94243766 = NA) OR (annual_ret#94243766 = null)) THEN null ELSE cast(annual_ret#94243766 as float) END AS annual_ret#94243868, CASE WHEN ((std_ret#94243767 = NA) OR (std_ret#94243767 = null)) THEN null ELSE cast(std_ret#94243767 as float) END AS std_ret#94243869, CASE WHEN ((Sharpe_ret#94243768 = NA) OR (Sharpe_ret#94243768 = null)) THEN null ELSE cast(Sharpe_ret#94243768 as float) END AS Sharpe_ret#94243870, CASE WHEN ((PctPos_ret#94243769 = NA) OR (PctPos_ret#94243769 = null)) THEN null ELSE cast(PctPos_ret#94243769 as float) END AS PctPos_ret#94243871, CASE WHEN ((TR_ret#94243770 = NA) OR (TR_ret#94243770 = null)) THEN null ELSE cast(TR_ret#94243770 as float) END AS TR_ret#94243872, CASE WHEN ((IR_ret#94243771 = NA) OR (IR_ret#94243771 = null)) THEN null ELSE cast(IR_ret#94243771 as float) END AS IR_ret#94243873, CASE WHEN ((annual_resret#94243772 = NA) OR (annual_resret#94243772 = null)) THEN null ELSE cast(annual_resret#94243772 as float) END AS annual_resret#94243907, CASE WHEN ((std_resret#94243773 = NA) OR (std_resret#94243773 = null)) THEN null ELSE cast(std_resret#94243773 as float) END AS std_resret#94243908, CASE WHEN ((Sharpe_resret#94243774 = NA) OR (Sharpe_resret#94243774 = null)) THEN null ELSE cast(Sharpe_resret#94243774 as float) END AS Sharpe_resret#94243909, CASE WHEN ((PctPos_resret#94243775 = NA) OR (PctPos_resret#94243775 = null)) THEN null ELSE cast(PctPos_resret#94243775 as float) END AS PctPos_resret#94243943, CASE WHEN ((TR_resret#94243776 = NA) OR (TR_resret#94243776 = null)) THEN null ELSE cast(TR_resret#94243776 as float) END AS TR_resret#94243944, CASE WHEN ((IR_resret#94243777 = NA) OR (IR_resret#94243777 = null)) THEN null ELSE cast(IR_resret#94243777 as float) END AS IR_resret#94243945, CASE WHEN ((annual_retnet#94243778 = NA) OR (annual_retnet#94243778 = null)) THEN null ELSE cast(annual_retnet#94243778 as float) END AS annual_retnet#94243946, CASE WHEN ((std_retnet#94243779 = NA) OR (std_retnet#94243779 = null)) THEN null ELSE cast(std_retnet#94243779 as float) END AS std_retnet#94243980, CASE WHEN ((Sharpe_retnet#94243780 = NA) OR (Sharpe_retnet#94243780 = null)) THEN null ELSE cast(Sharpe_retnet#94243780 as float) END AS Sharpe_retnet#94243981, CASE WHEN ((PctPos_retnet#94243781 = NA) OR (PctPos_retnet#94243781 = null)) THEN null ELSE cast(PctPos_retnet#94243781 as float) END AS PctPos_retnet#94243982, CASE WHEN ((TR_retnet#94243782 = NA) OR (TR_retnet#94243782 = null)) THEN null ELSE cast(TR_retnet#94243782 as float) END AS TR_retnet#94243983, CASE WHEN ((IR_retnet#94243783 = NA) OR (IR_retnet#94243783 = null)) THEN null ELSE cast(IR_retnet#94243783 as float) END AS IR_retnet#94243984, CASE WHEN ((turnover#94243784 = NA) OR (turnover#94243784 = null)) THEN null ELSE cast(turnover#94243784 as float) END AS turnover#94244018] Input [25]: [year#94243760, retIC#94243761, resretIC#94243762, numcos#94243763, numdates#94243764, annual_bmret#94243765, annual_ret#94243766, std_ret#94243767, Sharpe_ret#94243768, PctPos_ret#94243769, TR_ret#94243770, IR_ret#94243771, annual_resret#94243772, std_resret#94243773, Sharpe_resret#94243774, PctPos_resret#94243775, TR_resret#94243776, IR_resret#94243777, annual_retnet#94243778, std_retnet#94243779, Sharpe_retnet#94243780, PctPos_retnet#94243781, TR_retnet#94243782, IR_retnet#94243783, turnover#94243784] (7) ColumnarToRow [codegen id : 1] Input [2]: [turnover#94244018, year#94243861] (8) Filter [codegen id : 1] Input [2]: [turnover#94244018, year#94243861] Condition : isnotnull(turnover#94244018) (9) Project [codegen id : 1] Output [3]: [year#94243861, turnover#94244018, (1.0 / cast(turnover#94244018 as double)) AS days_hold#94244065] Input [2]: [turnover#94244018, year#94243861] (10) Exchange Input [3]: [year#94243861, turnover#94244018, days_hold#94244065] Arguments: rangepartitioning(year#94243861 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7524462] (11) Sort [codegen id : 2] Input [3]: [year#94243861, turnover#94244018, days_hold#94244065] Arguments: [year#94243861 ASC NULLS FIRST], true, 0 (12) ColumnarToRow [codegen id : 1] Input [3]: [year#94243861, turnover#94244018, days_hold#94244065] (13) CollectLimit Input [3]: [year#94243861, turnover#94244018, days_hold#94244065] Arguments: 1000000