== Physical Plan == CollectLimit (13) +- * ColumnarToRow (12) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (11) +- Exchange (10) +- * Project (9) +- * Filter (8) +- * ColumnarToRow (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [year#94385518, turnover#94385573, days_hold#94385633] Arguments: [year#94385518, turnover#94385573, days_hold#94385633] (2) InMemoryRelation Arguments: [year#94385518, turnover#94385573, days_hold#94385633], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [year#94385518 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94385518 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535880] +- *(1) Project [year#94385518, turnover#94385573, (1.0 / cast(turnover#94385573 as double)) AS days_hold#94385633] +- *(1) Filter isnotnull(turnover#94385573) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94385573, year#94385518], [isnotnull(turnover#94385573)] +- InMemoryRelation [year#94385518, retIC#94385520, resretIC#94385523, numcos#94385525, numdates#94385527, annual_bmret#94385530, annual_ret#94385532, std_ret#94385534, Sharpe_ret#94385536, PctPos_ret#94385539, TR_ret#94385541, IR_ret#94385543, annual_resret#94385545, std_resret#94385548, Sharpe_resret#94385550, PctPos_resret#94385552, TR_resret#94385555, IR_resret#94385557, annual_retnet#94385559, std_retnet#94385561, Sharpe_retnet#94385563, PctPos_retnet#94385566, TR_retnet#94385568, IR_retnet#94385571, turnover#94385573], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN ((year#94385350 = NA) OR (year#94385350 = null)) THEN null ELSE cast(year#94385350 as int) END AS year#94385518, CASE WHEN ((retIC#94385351 = NA) OR (retIC#94385351 = null)) THEN null ELSE cast(retIC#94385351 as float) END AS retIC#94385520, CASE WHEN ((resretIC#94385352 = NA) OR (resretIC#94385352 = null)) THEN null ELSE cast(resretIC#94385352 as float) END AS resretIC#94385523, CASE WHEN ((numcos#94385353 = NA) OR (numcos#94385353 = null)) THEN null ELSE cast(numcos#94385353 as float) END AS numcos#94385525, CASE WHEN ((numdates#94385354 = NA) OR (numdates#94385354 = null)) THEN null ELSE cast(numdates#94385354 as int) END AS numdates#94385527, CASE WHEN ((annual_bmret#94385355 = NA) OR (annual_bmret#94385355 = null)) THEN null ELSE cast(annual_bmret#94385355 as float) END AS annual_bmret#94385530, CASE WHEN ((annual_ret#94385356 = NA) OR (annual_ret#94385356 = null)) THEN null ELSE cast(annual_ret#94385356 as float) END AS annual_ret#94385532, CASE WHEN ((std_ret#94385357 = NA) OR (std_ret#94385357 = null)) THEN null ELSE cast(std_ret#94385357 as float) END AS std_ret#94385534, CASE WHEN ((Sharpe_ret#94385358 = NA) OR (Sharpe_ret#94385358 = null)) THEN null ELSE cast(Sharpe_ret#94385358 as float) END AS Sharpe_ret#94385536, CASE WHEN ((PctPos_ret#94385359 = NA) OR (PctPos_ret#94385359 = null)) THEN null ELSE cast(PctPos_ret#94385359 as float) END AS PctPos_ret#94385539, CASE WHEN ((TR_ret#94385360 = NA) OR (TR_ret#94385360 = null)) THEN null ELSE cast(TR_ret#94385360 as float) END AS TR_ret#94385541, CASE WHEN ((IR_ret#94385361 = NA) OR (IR_ret#94385361 = null)) THEN null ELSE cast(IR_ret#94385361 as float) END AS IR_ret#94385543, CASE WHEN ((annual_resret#94385362 = NA) OR (annual_resret#94385362 = null)) THEN null ELSE cast(annual_resret#94385362 as float) END AS annual_resret#94385545, CASE WHEN ((std_resret#94385363 = NA) OR (std_resret#94385363 = null)) THEN null ELSE cast(std_resret#94385363 as float) END AS std_resret#94385548, CASE WHEN ((Sharpe_resret#94385364 = NA) OR (Sharpe_resret#94385364 = null)) THEN null ELSE cast(Sharpe_resret#94385364 as float) END AS Sharpe_resret#94385550, CASE WHEN ((PctPos_resret#94385365 = NA) OR (PctPos_resret#94385365 = null)) THEN null ELSE cast(PctPos_resret#94385365 as float) END AS PctPos_resret#94385552, CASE WHEN ((TR_resret#94385366 = NA) OR (TR_resret#94385366 = null)) THEN null ELSE cast(TR_resret#94385366 as float) END AS TR_resret#94385555, CASE WHEN ((IR_resret#94385367 = NA) OR (IR_resret#94385367 = null)) THEN null ELSE cast(IR_resret#94385367 as float) END AS IR_resret#94385557, CASE WHEN ((annual_retnet#94385368 = NA) OR (annual_retnet#94385368 = null)) THEN null ELSE cast(annual_retnet#94385368 as float) END AS annual_retnet#94385559, CASE WHEN ((std_retnet#94385369 = NA) OR (std_retnet#94385369 = null)) THEN null ELSE cast(std_retnet#94385369 as float) END AS std_retnet#94385561, CASE WHEN ((Sharpe_retnet#94385370 = NA) OR (Sharpe_retnet#94385370 = null)) THEN null ELSE cast(Sharpe_retnet#94385370 as float) END AS Sharpe_retnet#94385563, CASE WHEN ((PctPos_retnet#94385371 = NA) OR (PctPos_retnet#94385371 = null)) THEN null ELSE cast(PctPos_retnet#94385371 as float) END AS PctPos_retnet#94385566, CASE WHEN ((TR_retnet#94385372 = NA) OR (TR_retnet#94385372 = null)) THEN null ELSE cast(TR_retnet#94385372 as float) END AS TR_retnet#94385568, CASE WHEN ((IR_retnet#94385373 = NA) OR (IR_retnet#94385373 = null)) THEN null ELSE cast(IR_retnet#94385373 as float) END AS IR_retnet#94385571, CASE WHEN ((turnover#94385374 = NA) OR (turnover#94385374 = null)) THEN null ELSE cast(turnover#94385374 as float) END AS turnover#94385573] +- FileScan csv [year#94385350,retIC#94385351,resretIC#94385352,numcos#94385353,numdates#94385354,annual_bmret#94385355,annual_ret#94385356,std_ret#94385357,Sharpe_ret#94385358,PctPos_ret#94385359,TR_ret#94385360,IR_ret#94385361,annual_resret#94385362,std_resret#94385363,Sharpe_resret#94385364,PctPos_resret#94385365,TR_resret#94385366,IR_resret#94385367,annual_retnet#94385368,std_retnet#94385369,Sharpe_retnet#94385370,PctPos_retnet#94385371,TR_retnet#94385372,IR_retnet#94385373,turnover#94385374] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None), [year#94385518 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [turnover#94385573, year#94385518] Arguments: [turnover#94385573, year#94385518], [isnotnull(turnover#94385573)] (4) InMemoryRelation Arguments: [year#94385518, retIC#94385520, resretIC#94385523, numcos#94385525, numdates#94385527, annual_bmret#94385530, annual_ret#94385532, std_ret#94385534, Sharpe_ret#94385536, PctPos_ret#94385539, TR_ret#94385541, IR_ret#94385543, annual_resret#94385545, std_resret#94385548, Sharpe_resret#94385550, PctPos_resret#94385552, TR_resret#94385555, IR_resret#94385557, annual_retnet#94385559, std_retnet#94385561, Sharpe_retnet#94385563, PctPos_retnet#94385566, TR_retnet#94385568, IR_retnet#94385571, turnover#94385573], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((year#94385350 = NA) OR (year#94385350 = null)) THEN null ELSE cast(year#94385350 as int) END AS year#94385518, CASE WHEN ((retIC#94385351 = NA) OR (retIC#94385351 = null)) THEN null ELSE cast(retIC#94385351 as float) END AS retIC#94385520, CASE WHEN ((resretIC#94385352 = NA) OR (resretIC#94385352 = null)) THEN null ELSE cast(resretIC#94385352 as float) END AS resretIC#94385523, CASE WHEN ((numcos#94385353 = NA) OR (numcos#94385353 = null)) THEN null ELSE cast(numcos#94385353 as float) END AS numcos#94385525, CASE WHEN ((numdates#94385354 = NA) OR (numdates#94385354 = null)) THEN null ELSE cast(numdates#94385354 as int) END AS numdates#94385527, CASE WHEN ((annual_bmret#94385355 = NA) OR (annual_bmret#94385355 = null)) THEN null ELSE cast(annual_bmret#94385355 as float) END AS annual_bmret#94385530, CASE WHEN ((annual_ret#94385356 = NA) OR (annual_ret#94385356 = null)) THEN null ELSE cast(annual_ret#94385356 as float) END AS annual_ret#94385532, CASE WHEN ((std_ret#94385357 = NA) OR (std_ret#94385357 = null)) THEN null ELSE cast(std_ret#94385357 as float) END AS std_ret#94385534, CASE WHEN ((Sharpe_ret#94385358 = NA) OR (Sharpe_ret#94385358 = null)) THEN null ELSE cast(Sharpe_ret#94385358 as float) END AS Sharpe_ret#94385536, CASE WHEN ((PctPos_ret#94385359 = NA) OR (PctPos_ret#94385359 = null)) THEN null ELSE cast(PctPos_ret#94385359 as float) END AS PctPos_ret#94385539, CASE WHEN ((TR_ret#94385360 = NA) OR (TR_ret#94385360 = null)) THEN null ELSE cast(TR_ret#94385360 as float) END AS TR_ret#94385541, CASE WHEN ((IR_ret#94385361 = NA) OR (IR_ret#94385361 = null)) THEN null ELSE cast(IR_ret#94385361 as float) END AS IR_ret#94385543, CASE WHEN ((annual_resret#94385362 = NA) OR (annual_resret#94385362 = null)) THEN null ELSE cast(annual_resret#94385362 as float) END AS annual_resret#94385545, CASE WHEN ((std_resret#94385363 = NA) OR (std_resret#94385363 = null)) THEN null ELSE cast(std_resret#94385363 as float) END AS std_resret#94385548, CASE WHEN ((Sharpe_resret#94385364 = NA) OR (Sharpe_resret#94385364 = null)) THEN null ELSE cast(Sharpe_resret#94385364 as float) END AS Sharpe_resret#94385550, CASE WHEN ((PctPos_resret#94385365 = NA) OR (PctPos_resret#94385365 = null)) THEN null ELSE cast(PctPos_resret#94385365 as float) END AS PctPos_resret#94385552, CASE WHEN ((TR_resret#94385366 = NA) OR (TR_resret#94385366 = null)) THEN null ELSE cast(TR_resret#94385366 as float) END AS TR_resret#94385555, CASE WHEN ((IR_resret#94385367 = NA) OR (IR_resret#94385367 = null)) THEN null ELSE cast(IR_resret#94385367 as float) END AS IR_resret#94385557, CASE WHEN ((annual_retnet#94385368 = NA) OR (annual_retnet#94385368 = null)) THEN null ELSE cast(annual_retnet#94385368 as float) END AS annual_retnet#94385559, CASE WHEN ((std_retnet#94385369 = NA) OR (std_retnet#94385369 = null)) THEN null ELSE cast(std_retnet#94385369 as float) END AS std_retnet#94385561, CASE WHEN ((Sharpe_retnet#94385370 = NA) OR (Sharpe_retnet#94385370 = null)) THEN null ELSE cast(Sharpe_retnet#94385370 as float) END AS Sharpe_retnet#94385563, CASE WHEN ((PctPos_retnet#94385371 = NA) OR (PctPos_retnet#94385371 = null)) THEN null ELSE cast(PctPos_retnet#94385371 as float) END AS PctPos_retnet#94385566, CASE WHEN ((TR_retnet#94385372 = NA) OR (TR_retnet#94385372 = null)) THEN null ELSE cast(TR_retnet#94385372 as float) END AS TR_retnet#94385568, CASE WHEN ((IR_retnet#94385373 = NA) OR (IR_retnet#94385373 = null)) THEN null ELSE cast(IR_retnet#94385373 as float) END AS IR_retnet#94385571, CASE WHEN ((turnover#94385374 = NA) OR (turnover#94385374 = null)) THEN null ELSE cast(turnover#94385374 as float) END AS turnover#94385573] +- FileScan csv [year#94385350,retIC#94385351,resretIC#94385352,numcos#94385353,numdates#94385354,annual_bmret#94385355,annual_ret#94385356,std_ret#94385357,Sharpe_ret#94385358,PctPos_ret#94385359,TR_ret#94385360,IR_ret#94385361,annual_resret#94385362,std_resret#94385363,Sharpe_resret#94385364,PctPos_resret#94385365,TR_resret#94385366,IR_resret#94385367,annual_retnet#94385368,std_retnet#94385369,Sharpe_retnet#94385370,PctPos_retnet#94385371,TR_retnet#94385372,IR_retnet#94385373,turnover#94385374] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None) (5) Scan csv Output [25]: [year#94385350, retIC#94385351, resretIC#94385352, numcos#94385353, numdates#94385354, annual_bmret#94385355, annual_ret#94385356, std_ret#94385357, Sharpe_ret#94385358, PctPos_ret#94385359, TR_ret#94385360, IR_ret#94385361, annual_resret#94385362, std_resret#94385363, Sharpe_resret#94385364, PctPos_resret#94385365, TR_resret#94385366, IR_resret#94385367, annual_retnet#94385368, std_retnet#94385369, Sharpe_retnet#94385370, PctPos_retnet#94385371, TR_retnet#94385372, IR_retnet#94385373, turnover#94385374] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/stats_year.csv] ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string> (6) Project [codegen id : 1] Output [25]: [CASE WHEN ((year#94385350 = NA) OR (year#94385350 = null)) THEN null ELSE cast(year#94385350 as int) END AS year#94385518, CASE WHEN ((retIC#94385351 = NA) OR (retIC#94385351 = null)) THEN null ELSE cast(retIC#94385351 as float) END AS retIC#94385520, CASE WHEN ((resretIC#94385352 = NA) OR (resretIC#94385352 = null)) THEN null ELSE cast(resretIC#94385352 as float) END AS resretIC#94385523, CASE WHEN ((numcos#94385353 = NA) OR (numcos#94385353 = null)) THEN null ELSE cast(numcos#94385353 as float) END AS numcos#94385525, CASE WHEN ((numdates#94385354 = NA) OR (numdates#94385354 = null)) THEN null ELSE cast(numdates#94385354 as int) END AS numdates#94385527, CASE WHEN ((annual_bmret#94385355 = NA) OR (annual_bmret#94385355 = null)) THEN null ELSE cast(annual_bmret#94385355 as float) END AS annual_bmret#94385530, CASE WHEN ((annual_ret#94385356 = NA) OR (annual_ret#94385356 = null)) THEN null ELSE cast(annual_ret#94385356 as float) END AS annual_ret#94385532, CASE WHEN ((std_ret#94385357 = NA) OR (std_ret#94385357 = null)) THEN null ELSE cast(std_ret#94385357 as float) END AS std_ret#94385534, CASE WHEN ((Sharpe_ret#94385358 = NA) OR (Sharpe_ret#94385358 = null)) THEN null ELSE cast(Sharpe_ret#94385358 as float) END AS Sharpe_ret#94385536, CASE WHEN ((PctPos_ret#94385359 = NA) OR (PctPos_ret#94385359 = null)) THEN null ELSE cast(PctPos_ret#94385359 as float) END AS PctPos_ret#94385539, CASE WHEN ((TR_ret#94385360 = NA) OR (TR_ret#94385360 = null)) THEN null ELSE cast(TR_ret#94385360 as float) END AS TR_ret#94385541, CASE WHEN ((IR_ret#94385361 = NA) OR (IR_ret#94385361 = null)) THEN null ELSE cast(IR_ret#94385361 as float) END AS IR_ret#94385543, CASE WHEN ((annual_resret#94385362 = NA) OR (annual_resret#94385362 = null)) THEN null ELSE cast(annual_resret#94385362 as float) END AS annual_resret#94385545, CASE WHEN ((std_resret#94385363 = NA) OR (std_resret#94385363 = null)) THEN null ELSE cast(std_resret#94385363 as float) END AS std_resret#94385548, CASE WHEN ((Sharpe_resret#94385364 = NA) OR (Sharpe_resret#94385364 = null)) THEN null ELSE cast(Sharpe_resret#94385364 as float) END AS Sharpe_resret#94385550, CASE WHEN ((PctPos_resret#94385365 = NA) OR (PctPos_resret#94385365 = null)) THEN null ELSE cast(PctPos_resret#94385365 as float) END AS PctPos_resret#94385552, CASE WHEN ((TR_resret#94385366 = NA) OR (TR_resret#94385366 = null)) THEN null ELSE cast(TR_resret#94385366 as float) END AS TR_resret#94385555, CASE WHEN ((IR_resret#94385367 = NA) OR (IR_resret#94385367 = null)) THEN null ELSE cast(IR_resret#94385367 as float) END AS IR_resret#94385557, CASE WHEN ((annual_retnet#94385368 = NA) OR (annual_retnet#94385368 = null)) THEN null ELSE cast(annual_retnet#94385368 as float) END AS annual_retnet#94385559, CASE WHEN ((std_retnet#94385369 = NA) OR (std_retnet#94385369 = null)) THEN null ELSE cast(std_retnet#94385369 as float) END AS std_retnet#94385561, CASE WHEN ((Sharpe_retnet#94385370 = NA) OR (Sharpe_retnet#94385370 = null)) THEN null ELSE cast(Sharpe_retnet#94385370 as float) END AS Sharpe_retnet#94385563, CASE WHEN ((PctPos_retnet#94385371 = NA) OR (PctPos_retnet#94385371 = null)) THEN null ELSE cast(PctPos_retnet#94385371 as float) END AS PctPos_retnet#94385566, CASE WHEN ((TR_retnet#94385372 = NA) OR (TR_retnet#94385372 = null)) THEN null ELSE cast(TR_retnet#94385372 as float) END AS TR_retnet#94385568, CASE WHEN ((IR_retnet#94385373 = NA) OR (IR_retnet#94385373 = null)) THEN null ELSE cast(IR_retnet#94385373 as float) END AS IR_retnet#94385571, CASE WHEN ((turnover#94385374 = NA) OR (turnover#94385374 = null)) THEN null ELSE cast(turnover#94385374 as float) END AS turnover#94385573] Input [25]: [year#94385350, retIC#94385351, resretIC#94385352, numcos#94385353, numdates#94385354, annual_bmret#94385355, annual_ret#94385356, std_ret#94385357, Sharpe_ret#94385358, PctPos_ret#94385359, TR_ret#94385360, IR_ret#94385361, annual_resret#94385362, std_resret#94385363, Sharpe_resret#94385364, PctPos_resret#94385365, TR_resret#94385366, IR_resret#94385367, annual_retnet#94385368, std_retnet#94385369, Sharpe_retnet#94385370, PctPos_retnet#94385371, TR_retnet#94385372, IR_retnet#94385373, turnover#94385374] (7) ColumnarToRow [codegen id : 1] Input [2]: [turnover#94385573, year#94385518] (8) Filter [codegen id : 1] Input [2]: [turnover#94385573, year#94385518] Condition : isnotnull(turnover#94385573) (9) Project [codegen id : 1] Output [3]: [year#94385518, turnover#94385573, (1.0 / cast(turnover#94385573 as double)) AS days_hold#94385633] Input [2]: [turnover#94385573, year#94385518] (10) Exchange Input [3]: [year#94385518, turnover#94385573, days_hold#94385633] Arguments: rangepartitioning(year#94385518 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535880] (11) Sort [codegen id : 2] Input [3]: [year#94385518, turnover#94385573, days_hold#94385633] Arguments: [year#94385518 ASC NULLS FIRST], true, 0 (12) ColumnarToRow [codegen id : 1] Input [3]: [year#94385518, turnover#94385573, days_hold#94385633] (13) CollectLimit Input [3]: [year#94385518, turnover#94385573, days_hold#94385633] Arguments: 10000