== Physical Plan == CollectLimit (13) +- * ColumnarToRow (12) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (11) +- Exchange (10) +- * Project (9) +- * Filter (8) +- * ColumnarToRow (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [year#94114007, turnover#94114061, days_hold#94114121] Arguments: [year#94114007, turnover#94114061, days_hold#94114121] (2) InMemoryRelation Arguments: [year#94114007, turnover#94114061, days_hold#94114121], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [year#94114007 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94114007 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7514238] +- *(1) Project [year#94114007, turnover#94114061, (1.0 / cast(turnover#94114061 as double)) AS days_hold#94114121] +- *(1) Filter isnotnull(turnover#94114061) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94114061, year#94114007], [isnotnull(turnover#94114061)] +- InMemoryRelation [year#94114007, retIC#94114009, resretIC#94114011, numcos#94114013, numdates#94114015, annual_bmret#94114018, annual_ret#94114020, std_ret#94114022, Sharpe_ret#94114024, PctPos_ret#94114027, TR_ret#94114029, IR_ret#94114031, annual_resret#94114034, std_resret#94114036, Sharpe_resret#94114038, PctPos_resret#94114041, TR_resret#94114043, IR_resret#94114045, annual_retnet#94114048, std_retnet#94114050, Sharpe_retnet#94114052, PctPos_retnet#94114055, TR_retnet#94114057, IR_retnet#94114059, turnover#94114061], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN ((year#94113839 = NA) OR (year#94113839 = null)) THEN null ELSE cast(year#94113839 as float) END AS year#94114007, CASE WHEN ((retIC#94113840 = NA) OR (retIC#94113840 = null)) THEN null ELSE cast(retIC#94113840 as float) END AS retIC#94114009, CASE WHEN ((resretIC#94113841 = NA) OR (resretIC#94113841 = null)) THEN null ELSE cast(resretIC#94113841 as float) END AS resretIC#94114011, CASE WHEN ((numcos#94113842 = NA) OR (numcos#94113842 = null)) THEN null ELSE cast(numcos#94113842 as float) END AS numcos#94114013, CASE WHEN ((numdates#94113843 = NA) OR (numdates#94113843 = null)) THEN null ELSE cast(numdates#94113843 as int) END AS numdates#94114015, CASE WHEN ((annual_bmret#94113844 = NA) OR (annual_bmret#94113844 = null)) THEN null ELSE cast(annual_bmret#94113844 as float) END AS annual_bmret#94114018, CASE WHEN ((annual_ret#94113845 = NA) OR (annual_ret#94113845 = null)) THEN null ELSE cast(annual_ret#94113845 as float) END AS annual_ret#94114020, CASE WHEN ((std_ret#94113846 = NA) OR (std_ret#94113846 = null)) THEN null ELSE cast(std_ret#94113846 as float) END AS std_ret#94114022, CASE WHEN ((Sharpe_ret#94113847 = NA) OR (Sharpe_ret#94113847 = null)) THEN null ELSE cast(Sharpe_ret#94113847 as float) END AS Sharpe_ret#94114024, CASE WHEN ((PctPos_ret#94113848 = NA) OR (PctPos_ret#94113848 = null)) THEN null ELSE cast(PctPos_ret#94113848 as float) END AS PctPos_ret#94114027, CASE WHEN ((TR_ret#94113849 = NA) OR (TR_ret#94113849 = null)) THEN null ELSE cast(TR_ret#94113849 as float) END AS TR_ret#94114029, CASE WHEN ((IR_ret#94113850 = NA) OR (IR_ret#94113850 = null)) THEN null ELSE cast(IR_ret#94113850 as float) END AS IR_ret#94114031, CASE WHEN ((annual_resret#94113851 = NA) OR (annual_resret#94113851 = null)) THEN null ELSE cast(annual_resret#94113851 as float) END AS annual_resret#94114034, CASE WHEN ((std_resret#94113852 = NA) OR (std_resret#94113852 = null)) THEN null ELSE cast(std_resret#94113852 as float) END AS std_resret#94114036, CASE WHEN ((Sharpe_resret#94113853 = NA) OR (Sharpe_resret#94113853 = null)) THEN null ELSE cast(Sharpe_resret#94113853 as float) END AS Sharpe_resret#94114038, CASE WHEN ((PctPos_resret#94113854 = NA) OR (PctPos_resret#94113854 = null)) THEN null ELSE cast(PctPos_resret#94113854 as float) END AS PctPos_resret#94114041, CASE WHEN ((TR_resret#94113855 = NA) OR (TR_resret#94113855 = null)) THEN null ELSE cast(TR_resret#94113855 as float) END AS TR_resret#94114043, CASE WHEN ((IR_resret#94113856 = NA) OR (IR_resret#94113856 = null)) THEN null ELSE cast(IR_resret#94113856 as float) END AS IR_resret#94114045, CASE WHEN ((annual_retnet#94113857 = NA) OR (annual_retnet#94113857 = null)) THEN null ELSE cast(annual_retnet#94113857 as float) END AS annual_retnet#94114048, CASE WHEN ((std_retnet#94113858 = NA) OR (std_retnet#94113858 = null)) THEN null ELSE cast(std_retnet#94113858 as float) END AS std_retnet#94114050, CASE WHEN ((Sharpe_retnet#94113859 = NA) OR (Sharpe_retnet#94113859 = null)) THEN null ELSE cast(Sharpe_retnet#94113859 as float) END AS Sharpe_retnet#94114052, CASE WHEN ((PctPos_retnet#94113860 = NA) OR (PctPos_retnet#94113860 = null)) THEN null ELSE cast(PctPos_retnet#94113860 as float) END AS PctPos_retnet#94114055, CASE WHEN ((TR_retnet#94113861 = NA) OR (TR_retnet#94113861 = null)) THEN null ELSE cast(TR_retnet#94113861 as float) END AS TR_retnet#94114057, CASE WHEN ((IR_retnet#94113862 = NA) OR (IR_retnet#94113862 = null)) THEN null ELSE cast(IR_retnet#94113862 as float) END AS IR_retnet#94114059, CASE WHEN ((turnover#94113863 = NA) OR (turnover#94113863 = null)) THEN null ELSE cast(turnover#94113863 as float) END AS turnover#94114061] +- FileScan csv [year#94113839,retIC#94113840,resretIC#94113841,numcos#94113842,numdates#94113843,annual_bmret#94113844,annual_ret#94113845,std_ret#94113846,Sharpe_ret#94113847,PctPos_ret#94113848,TR_ret#94113849,IR_ret#94113850,annual_resret#94113851,std_resret#94113852,Sharpe_resret#94113853,PctPos_resret#94113854,TR_resret#94113855,IR_resret#94113856,annual_retnet#94113857,std_retnet#94113858,Sharpe_retnet#94113859,PctPos_retnet#94113860,TR_retnet#94113861,IR_retnet#94113862,turnover#94113863] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/tm1/eatm1_score/stats_..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None), [year#94114007 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [turnover#94114061, year#94114007] Arguments: [turnover#94114061, year#94114007], [isnotnull(turnover#94114061)] (4) InMemoryRelation Arguments: [year#94114007, retIC#94114009, resretIC#94114011, numcos#94114013, numdates#94114015, annual_bmret#94114018, annual_ret#94114020, std_ret#94114022, Sharpe_ret#94114024, PctPos_ret#94114027, TR_ret#94114029, IR_ret#94114031, annual_resret#94114034, std_resret#94114036, Sharpe_resret#94114038, PctPos_resret#94114041, TR_resret#94114043, IR_resret#94114045, annual_retnet#94114048, std_retnet#94114050, Sharpe_retnet#94114052, PctPos_retnet#94114055, TR_retnet#94114057, IR_retnet#94114059, turnover#94114061], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((year#94113839 = NA) OR (year#94113839 = null)) THEN null ELSE cast(year#94113839 as float) END AS year#94114007, CASE WHEN ((retIC#94113840 = NA) OR (retIC#94113840 = null)) THEN null ELSE cast(retIC#94113840 as float) END AS retIC#94114009, CASE WHEN ((resretIC#94113841 = NA) OR (resretIC#94113841 = null)) THEN null ELSE cast(resretIC#94113841 as float) END AS resretIC#94114011, CASE WHEN ((numcos#94113842 = NA) OR (numcos#94113842 = null)) THEN null ELSE cast(numcos#94113842 as float) END AS numcos#94114013, CASE WHEN ((numdates#94113843 = NA) OR (numdates#94113843 = null)) THEN null ELSE cast(numdates#94113843 as int) END AS numdates#94114015, CASE WHEN ((annual_bmret#94113844 = NA) OR (annual_bmret#94113844 = null)) THEN null ELSE cast(annual_bmret#94113844 as float) END AS annual_bmret#94114018, CASE WHEN ((annual_ret#94113845 = NA) OR (annual_ret#94113845 = null)) THEN null ELSE cast(annual_ret#94113845 as float) END AS annual_ret#94114020, CASE WHEN ((std_ret#94113846 = NA) OR (std_ret#94113846 = null)) THEN null ELSE cast(std_ret#94113846 as float) END AS std_ret#94114022, CASE WHEN ((Sharpe_ret#94113847 = NA) OR (Sharpe_ret#94113847 = null)) THEN null ELSE cast(Sharpe_ret#94113847 as float) END AS Sharpe_ret#94114024, CASE WHEN ((PctPos_ret#94113848 = NA) OR (PctPos_ret#94113848 = null)) THEN null ELSE cast(PctPos_ret#94113848 as float) END AS PctPos_ret#94114027, CASE WHEN ((TR_ret#94113849 = NA) OR (TR_ret#94113849 = null)) THEN null ELSE cast(TR_ret#94113849 as float) END AS TR_ret#94114029, CASE WHEN ((IR_ret#94113850 = NA) OR (IR_ret#94113850 = null)) THEN null ELSE cast(IR_ret#94113850 as float) END AS IR_ret#94114031, CASE WHEN ((annual_resret#94113851 = NA) OR (annual_resret#94113851 = null)) THEN null ELSE cast(annual_resret#94113851 as float) END AS annual_resret#94114034, CASE WHEN ((std_resret#94113852 = NA) OR (std_resret#94113852 = null)) THEN null ELSE cast(std_resret#94113852 as float) END AS std_resret#94114036, CASE WHEN ((Sharpe_resret#94113853 = NA) OR (Sharpe_resret#94113853 = null)) THEN null ELSE cast(Sharpe_resret#94113853 as float) END AS Sharpe_resret#94114038, CASE WHEN ((PctPos_resret#94113854 = NA) OR (PctPos_resret#94113854 = null)) THEN null ELSE cast(PctPos_resret#94113854 as float) END AS PctPos_resret#94114041, CASE WHEN ((TR_resret#94113855 = NA) OR (TR_resret#94113855 = null)) THEN null ELSE cast(TR_resret#94113855 as float) END AS TR_resret#94114043, CASE WHEN ((IR_resret#94113856 = NA) OR (IR_resret#94113856 = null)) THEN null ELSE cast(IR_resret#94113856 as float) END AS IR_resret#94114045, CASE WHEN ((annual_retnet#94113857 = NA) OR (annual_retnet#94113857 = null)) THEN null ELSE cast(annual_retnet#94113857 as float) END AS annual_retnet#94114048, CASE WHEN ((std_retnet#94113858 = NA) OR (std_retnet#94113858 = null)) THEN null ELSE cast(std_retnet#94113858 as float) END AS std_retnet#94114050, CASE WHEN ((Sharpe_retnet#94113859 = NA) OR (Sharpe_retnet#94113859 = null)) THEN null ELSE cast(Sharpe_retnet#94113859 as float) END AS Sharpe_retnet#94114052, CASE WHEN ((PctPos_retnet#94113860 = NA) OR (PctPos_retnet#94113860 = null)) THEN null ELSE cast(PctPos_retnet#94113860 as float) END AS PctPos_retnet#94114055, CASE WHEN ((TR_retnet#94113861 = NA) OR (TR_retnet#94113861 = null)) THEN null ELSE cast(TR_retnet#94113861 as float) END AS TR_retnet#94114057, CASE WHEN ((IR_retnet#94113862 = NA) OR (IR_retnet#94113862 = null)) THEN null ELSE cast(IR_retnet#94113862 as float) END AS IR_retnet#94114059, CASE WHEN ((turnover#94113863 = NA) OR (turnover#94113863 = null)) THEN null ELSE cast(turnover#94113863 as float) END AS turnover#94114061] +- FileScan csv [year#94113839,retIC#94113840,resretIC#94113841,numcos#94113842,numdates#94113843,annual_bmret#94113844,annual_ret#94113845,std_ret#94113846,Sharpe_ret#94113847,PctPos_ret#94113848,TR_ret#94113849,IR_ret#94113850,annual_resret#94113851,std_resret#94113852,Sharpe_resret#94113853,PctPos_resret#94113854,TR_resret#94113855,IR_resret#94113856,annual_retnet#94113857,std_retnet#94113858,Sharpe_retnet#94113859,PctPos_retnet#94113860,TR_retnet#94113861,IR_retnet#94113862,turnover#94113863] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/tm1/eatm1_score/stats_..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None) (5) Scan csv Output [25]: [year#94113839, retIC#94113840, resretIC#94113841, numcos#94113842, numdates#94113843, annual_bmret#94113844, annual_ret#94113845, std_ret#94113846, Sharpe_ret#94113847, PctPos_ret#94113848, TR_ret#94113849, IR_ret#94113850, annual_resret#94113851, std_resret#94113852, Sharpe_resret#94113853, PctPos_resret#94113854, TR_resret#94113855, IR_resret#94113856, annual_retnet#94113857, std_retnet#94113858, Sharpe_retnet#94113859, PctPos_retnet#94113860, TR_retnet#94113861, IR_retnet#94113862, turnover#94113863] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/tm1/eatm1_score/stats_year.csv] ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string> (6) Project [codegen id : 1] Output [25]: [CASE WHEN ((year#94113839 = NA) OR (year#94113839 = null)) THEN null ELSE cast(year#94113839 as float) END AS year#94114007, CASE WHEN ((retIC#94113840 = NA) OR (retIC#94113840 = null)) THEN null ELSE cast(retIC#94113840 as float) END AS retIC#94114009, CASE WHEN ((resretIC#94113841 = NA) OR (resretIC#94113841 = null)) THEN null ELSE cast(resretIC#94113841 as float) END AS resretIC#94114011, CASE WHEN ((numcos#94113842 = NA) OR (numcos#94113842 = null)) THEN null ELSE cast(numcos#94113842 as float) END AS numcos#94114013, CASE WHEN ((numdates#94113843 = NA) OR (numdates#94113843 = null)) THEN null ELSE cast(numdates#94113843 as int) END AS numdates#94114015, CASE WHEN ((annual_bmret#94113844 = NA) OR (annual_bmret#94113844 = null)) THEN null ELSE cast(annual_bmret#94113844 as float) END AS annual_bmret#94114018, CASE WHEN ((annual_ret#94113845 = NA) OR (annual_ret#94113845 = null)) THEN null ELSE cast(annual_ret#94113845 as float) END AS annual_ret#94114020, CASE WHEN ((std_ret#94113846 = NA) OR (std_ret#94113846 = null)) THEN null ELSE cast(std_ret#94113846 as float) END AS std_ret#94114022, CASE WHEN ((Sharpe_ret#94113847 = NA) OR (Sharpe_ret#94113847 = null)) THEN null ELSE cast(Sharpe_ret#94113847 as float) END AS Sharpe_ret#94114024, CASE WHEN ((PctPos_ret#94113848 = NA) OR (PctPos_ret#94113848 = null)) THEN null ELSE cast(PctPos_ret#94113848 as float) END AS PctPos_ret#94114027, CASE WHEN ((TR_ret#94113849 = NA) OR (TR_ret#94113849 = null)) THEN null ELSE cast(TR_ret#94113849 as float) END AS TR_ret#94114029, CASE WHEN ((IR_ret#94113850 = NA) OR (IR_ret#94113850 = null)) THEN null ELSE cast(IR_ret#94113850 as float) END AS IR_ret#94114031, CASE WHEN ((annual_resret#94113851 = NA) OR (annual_resret#94113851 = null)) THEN null ELSE cast(annual_resret#94113851 as float) END AS annual_resret#94114034, CASE WHEN ((std_resret#94113852 = NA) OR (std_resret#94113852 = null)) THEN null ELSE cast(std_resret#94113852 as float) END AS std_resret#94114036, CASE WHEN ((Sharpe_resret#94113853 = NA) OR (Sharpe_resret#94113853 = null)) THEN null ELSE cast(Sharpe_resret#94113853 as float) END AS Sharpe_resret#94114038, CASE WHEN ((PctPos_resret#94113854 = NA) OR (PctPos_resret#94113854 = null)) THEN null ELSE cast(PctPos_resret#94113854 as float) END AS PctPos_resret#94114041, CASE WHEN ((TR_resret#94113855 = NA) OR (TR_resret#94113855 = null)) THEN null ELSE cast(TR_resret#94113855 as float) END AS TR_resret#94114043, CASE WHEN ((IR_resret#94113856 = NA) OR (IR_resret#94113856 = null)) THEN null ELSE cast(IR_resret#94113856 as float) END AS IR_resret#94114045, CASE WHEN ((annual_retnet#94113857 = NA) OR (annual_retnet#94113857 = null)) THEN null ELSE cast(annual_retnet#94113857 as float) END AS annual_retnet#94114048, CASE WHEN ((std_retnet#94113858 = NA) OR (std_retnet#94113858 = null)) THEN null ELSE cast(std_retnet#94113858 as float) END AS std_retnet#94114050, CASE WHEN ((Sharpe_retnet#94113859 = NA) OR (Sharpe_retnet#94113859 = null)) THEN null ELSE cast(Sharpe_retnet#94113859 as float) END AS Sharpe_retnet#94114052, CASE WHEN ((PctPos_retnet#94113860 = NA) OR (PctPos_retnet#94113860 = null)) THEN null ELSE cast(PctPos_retnet#94113860 as float) END AS PctPos_retnet#94114055, CASE WHEN ((TR_retnet#94113861 = NA) OR (TR_retnet#94113861 = null)) THEN null ELSE cast(TR_retnet#94113861 as float) END AS TR_retnet#94114057, CASE WHEN ((IR_retnet#94113862 = NA) OR (IR_retnet#94113862 = null)) THEN null ELSE cast(IR_retnet#94113862 as float) END AS IR_retnet#94114059, CASE WHEN ((turnover#94113863 = NA) OR (turnover#94113863 = null)) THEN null ELSE cast(turnover#94113863 as float) END AS turnover#94114061] Input [25]: [year#94113839, retIC#94113840, resretIC#94113841, numcos#94113842, numdates#94113843, annual_bmret#94113844, annual_ret#94113845, std_ret#94113846, Sharpe_ret#94113847, PctPos_ret#94113848, TR_ret#94113849, IR_ret#94113850, annual_resret#94113851, std_resret#94113852, Sharpe_resret#94113853, PctPos_resret#94113854, TR_resret#94113855, IR_resret#94113856, annual_retnet#94113857, std_retnet#94113858, Sharpe_retnet#94113859, PctPos_retnet#94113860, TR_retnet#94113861, IR_retnet#94113862, turnover#94113863] (7) ColumnarToRow [codegen id : 1] Input [2]: [turnover#94114061, year#94114007] (8) Filter [codegen id : 1] Input [2]: [turnover#94114061, year#94114007] Condition : isnotnull(turnover#94114061) (9) Project [codegen id : 1] Output [3]: [year#94114007, turnover#94114061, (1.0 / cast(turnover#94114061 as double)) AS days_hold#94114121] Input [2]: [turnover#94114061, year#94114007] (10) Exchange Input [3]: [year#94114007, turnover#94114061, days_hold#94114121] Arguments: rangepartitioning(year#94114007 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7514238] (11) Sort [codegen id : 2] Input [3]: [year#94114007, turnover#94114061, days_hold#94114121] Arguments: [year#94114007 ASC NULLS FIRST], true, 0 (12) ColumnarToRow [codegen id : 1] Input [3]: [year#94114007, turnover#94114061, days_hold#94114121] (13) CollectLimit Input [3]: [year#94114007, turnover#94114061, days_hold#94114121] Arguments: 10000