== Physical Plan == CollectLimit (13) +- * ColumnarToRow (12) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (11) +- Exchange (10) +- * Project (9) +- * Filter (8) +- * ColumnarToRow (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [year#93999692, turnover#93999740, days_hold#93999774] Arguments: [year#93999692, turnover#93999740, days_hold#93999774] (2) InMemoryRelation Arguments: [year#93999692, turnover#93999740, days_hold#93999774], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [year#93999692 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#93999692 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7505140] +- *(1) Project [year#93999692, turnover#93999740, (1.0 / cast(turnover#93999740 as double)) AS days_hold#93999774] +- *(1) Filter isnotnull(turnover#93999740) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#93999740, year#93999692], [isnotnull(turnover#93999740)] +- InMemoryRelation [year#93999692, retIC#93999694, resretIC#93999696, numcos#93999698, numdates#93999700, annual_bmret#93999702, annual_ret#93999704, std_ret#93999706, Sharpe_ret#93999708, PctPos_ret#93999710, TR_ret#93999712, IR_ret#93999714, annual_resret#93999716, std_resret#93999718, Sharpe_resret#93999720, PctPos_resret#93999722, TR_resret#93999724, IR_resret#93999726, annual_retnet#93999728, std_retnet#93999730, Sharpe_retnet#93999732, PctPos_retnet#93999734, TR_retnet#93999736, IR_retnet#93999738, turnover#93999740], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN ((year#93999459 = NA) OR (year#93999459 = null)) THEN null ELSE cast(year#93999459 as float) END AS year#93999692, CASE WHEN ((retIC#93999460 = NA) OR (retIC#93999460 = null)) THEN null ELSE cast(retIC#93999460 as float) END AS retIC#93999694, CASE WHEN ((resretIC#93999461 = NA) OR (resretIC#93999461 = null)) THEN null ELSE cast(resretIC#93999461 as float) END AS resretIC#93999696, CASE WHEN ((numcos#93999462 = NA) OR (numcos#93999462 = null)) THEN null ELSE cast(numcos#93999462 as float) END AS numcos#93999698, CASE WHEN ((numdates#93999463 = NA) OR (numdates#93999463 = null)) THEN null ELSE cast(numdates#93999463 as int) END AS numdates#93999700, CASE WHEN ((annual_bmret#93999464 = NA) OR (annual_bmret#93999464 = null)) THEN null ELSE cast(annual_bmret#93999464 as float) END AS annual_bmret#93999702, CASE WHEN ((annual_ret#93999465 = NA) OR (annual_ret#93999465 = null)) THEN null ELSE cast(annual_ret#93999465 as float) END AS annual_ret#93999704, CASE WHEN ((std_ret#93999466 = NA) OR (std_ret#93999466 = null)) THEN null ELSE cast(std_ret#93999466 as float) END AS std_ret#93999706, CASE WHEN ((Sharpe_ret#93999467 = NA) OR (Sharpe_ret#93999467 = null)) THEN null ELSE cast(Sharpe_ret#93999467 as float) END AS Sharpe_ret#93999708, CASE WHEN ((PctPos_ret#93999468 = NA) OR (PctPos_ret#93999468 = null)) THEN null ELSE cast(PctPos_ret#93999468 as float) END AS PctPos_ret#93999710, CASE WHEN ((TR_ret#93999469 = NA) OR (TR_ret#93999469 = null)) THEN null ELSE cast(TR_ret#93999469 as float) END AS TR_ret#93999712, CASE WHEN ((IR_ret#93999470 = NA) OR (IR_ret#93999470 = null)) THEN null ELSE cast(IR_ret#93999470 as float) END AS IR_ret#93999714, CASE WHEN ((annual_resret#93999471 = NA) OR (annual_resret#93999471 = null)) THEN null ELSE cast(annual_resret#93999471 as float) END AS annual_resret#93999716, CASE WHEN ((std_resret#93999472 = NA) OR (std_resret#93999472 = null)) THEN null ELSE cast(std_resret#93999472 as float) END AS std_resret#93999718, CASE WHEN ((Sharpe_resret#93999473 = NA) OR (Sharpe_resret#93999473 = null)) THEN null ELSE cast(Sharpe_resret#93999473 as float) END AS Sharpe_resret#93999720, CASE WHEN ((PctPos_resret#93999474 = NA) OR (PctPos_resret#93999474 = null)) THEN null ELSE cast(PctPos_resret#93999474 as float) END AS PctPos_resret#93999722, CASE WHEN ((TR_resret#93999475 = NA) OR (TR_resret#93999475 = null)) THEN null ELSE cast(TR_resret#93999475 as float) END AS TR_resret#93999724, CASE WHEN ((IR_resret#93999476 = NA) OR (IR_resret#93999476 = null)) THEN null ELSE cast(IR_resret#93999476 as float) END AS IR_resret#93999726, CASE WHEN ((annual_retnet#93999477 = NA) OR (annual_retnet#93999477 = null)) THEN null ELSE cast(annual_retnet#93999477 as float) END AS annual_retnet#93999728, CASE WHEN ((std_retnet#93999478 = NA) OR (std_retnet#93999478 = null)) THEN null ELSE cast(std_retnet#93999478 as float) END AS std_retnet#93999730, CASE WHEN ((Sharpe_retnet#93999479 = NA) OR (Sharpe_retnet#93999479 = null)) THEN null ELSE cast(Sharpe_retnet#93999479 as float) END AS Sharpe_retnet#93999732, CASE WHEN ((PctPos_retnet#93999480 = NA) OR (PctPos_retnet#93999480 = null)) THEN null ELSE cast(PctPos_retnet#93999480 as float) END AS PctPos_retnet#93999734, CASE WHEN ((TR_retnet#93999481 = NA) OR (TR_retnet#93999481 = null)) THEN null ELSE cast(TR_retnet#93999481 as float) END AS TR_retnet#93999736, CASE WHEN ((IR_retnet#93999482 = NA) OR (IR_retnet#93999482 = null)) THEN null ELSE cast(IR_retnet#93999482 as float) END AS IR_retnet#93999738, CASE WHEN ((turnover#93999483 = NA) OR (turnover#93999483 = null)) THEN null ELSE cast(turnover#93999483 as float) END AS turnover#93999740] +- FileScan csv [year#93999459,retIC#93999460,resretIC#93999461,numcos#93999462,numdates#93999463,annual_bmret#93999464,annual_ret#93999465,std_ret#93999466,Sharpe_ret#93999467,PctPos_ret#93999468,TR_ret#93999469,IR_ret#93999470,annual_resret#93999471,std_resret#93999472,Sharpe_resret#93999473,PctPos_resret#93999474,TR_resret#93999475,IR_resret#93999476,annual_retnet#93999477,std_retnet#93999478,Sharpe_retnet#93999479,PctPos_retnet#93999480,TR_retnet#93999481,IR_retnet#93999482,turnover#93999483] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None), [year#93999692 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [turnover#93999740, year#93999692] Arguments: [turnover#93999740, year#93999692], [isnotnull(turnover#93999740)] (4) InMemoryRelation Arguments: [year#93999692, retIC#93999694, resretIC#93999696, numcos#93999698, numdates#93999700, annual_bmret#93999702, annual_ret#93999704, std_ret#93999706, Sharpe_ret#93999708, PctPos_ret#93999710, TR_ret#93999712, IR_ret#93999714, annual_resret#93999716, std_resret#93999718, Sharpe_resret#93999720, PctPos_resret#93999722, TR_resret#93999724, IR_resret#93999726, annual_retnet#93999728, std_retnet#93999730, Sharpe_retnet#93999732, PctPos_retnet#93999734, TR_retnet#93999736, IR_retnet#93999738, turnover#93999740], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((year#93999459 = NA) OR (year#93999459 = null)) THEN null ELSE cast(year#93999459 as float) END AS year#93999692, CASE WHEN ((retIC#93999460 = NA) OR (retIC#93999460 = null)) THEN null ELSE cast(retIC#93999460 as float) END AS retIC#93999694, CASE WHEN ((resretIC#93999461 = NA) OR (resretIC#93999461 = null)) THEN null ELSE cast(resretIC#93999461 as float) END AS resretIC#93999696, CASE WHEN ((numcos#93999462 = NA) OR (numcos#93999462 = null)) THEN null ELSE cast(numcos#93999462 as float) END AS numcos#93999698, CASE WHEN ((numdates#93999463 = NA) OR (numdates#93999463 = null)) THEN null ELSE cast(numdates#93999463 as int) END AS numdates#93999700, CASE WHEN ((annual_bmret#93999464 = NA) OR (annual_bmret#93999464 = null)) THEN null ELSE cast(annual_bmret#93999464 as float) END AS annual_bmret#93999702, CASE WHEN ((annual_ret#93999465 = NA) OR (annual_ret#93999465 = null)) THEN null ELSE cast(annual_ret#93999465 as float) END AS annual_ret#93999704, CASE WHEN ((std_ret#93999466 = NA) OR (std_ret#93999466 = null)) THEN null ELSE cast(std_ret#93999466 as float) END AS std_ret#93999706, CASE WHEN ((Sharpe_ret#93999467 = NA) OR (Sharpe_ret#93999467 = null)) THEN null ELSE cast(Sharpe_ret#93999467 as float) END AS Sharpe_ret#93999708, CASE WHEN ((PctPos_ret#93999468 = NA) OR (PctPos_ret#93999468 = null)) THEN null ELSE cast(PctPos_ret#93999468 as float) END AS PctPos_ret#93999710, CASE WHEN ((TR_ret#93999469 = NA) OR (TR_ret#93999469 = null)) THEN null ELSE cast(TR_ret#93999469 as float) END AS TR_ret#93999712, CASE WHEN ((IR_ret#93999470 = NA) OR (IR_ret#93999470 = null)) THEN null ELSE cast(IR_ret#93999470 as float) END AS IR_ret#93999714, CASE WHEN ((annual_resret#93999471 = NA) OR (annual_resret#93999471 = null)) THEN null ELSE cast(annual_resret#93999471 as float) END AS annual_resret#93999716, CASE WHEN ((std_resret#93999472 = NA) OR (std_resret#93999472 = null)) THEN null ELSE cast(std_resret#93999472 as float) END AS std_resret#93999718, CASE WHEN ((Sharpe_resret#93999473 = NA) OR (Sharpe_resret#93999473 = null)) THEN null ELSE cast(Sharpe_resret#93999473 as float) END AS Sharpe_resret#93999720, CASE WHEN ((PctPos_resret#93999474 = NA) OR (PctPos_resret#93999474 = null)) THEN null ELSE cast(PctPos_resret#93999474 as float) END AS PctPos_resret#93999722, CASE WHEN ((TR_resret#93999475 = NA) OR (TR_resret#93999475 = null)) THEN null ELSE cast(TR_resret#93999475 as float) END AS TR_resret#93999724, CASE WHEN ((IR_resret#93999476 = NA) OR (IR_resret#93999476 = null)) THEN null ELSE cast(IR_resret#93999476 as float) END AS IR_resret#93999726, CASE WHEN ((annual_retnet#93999477 = NA) OR (annual_retnet#93999477 = null)) THEN null ELSE cast(annual_retnet#93999477 as float) END AS annual_retnet#93999728, CASE WHEN ((std_retnet#93999478 = NA) OR (std_retnet#93999478 = null)) THEN null ELSE cast(std_retnet#93999478 as float) END AS std_retnet#93999730, CASE WHEN ((Sharpe_retnet#93999479 = NA) OR (Sharpe_retnet#93999479 = null)) THEN null ELSE cast(Sharpe_retnet#93999479 as float) END AS Sharpe_retnet#93999732, CASE WHEN ((PctPos_retnet#93999480 = NA) OR (PctPos_retnet#93999480 = null)) THEN null ELSE cast(PctPos_retnet#93999480 as float) END AS PctPos_retnet#93999734, CASE WHEN ((TR_retnet#93999481 = NA) OR (TR_retnet#93999481 = null)) THEN null ELSE cast(TR_retnet#93999481 as float) END AS TR_retnet#93999736, CASE WHEN ((IR_retnet#93999482 = NA) OR (IR_retnet#93999482 = null)) THEN null ELSE cast(IR_retnet#93999482 as float) END AS IR_retnet#93999738, CASE WHEN ((turnover#93999483 = NA) OR (turnover#93999483 = null)) THEN null ELSE cast(turnover#93999483 as float) END AS turnover#93999740] +- FileScan csv [year#93999459,retIC#93999460,resretIC#93999461,numcos#93999462,numdates#93999463,annual_bmret#93999464,annual_ret#93999465,std_ret#93999466,Sharpe_ret#93999467,PctPos_ret#93999468,TR_ret#93999469,IR_ret#93999470,annual_resret#93999471,std_resret#93999472,Sharpe_resret#93999473,PctPos_resret#93999474,TR_resret#93999475,IR_resret#93999476,annual_retnet#93999477,std_retnet#93999478,Sharpe_retnet#93999479,PctPos_retnet#93999480,TR_retnet#93999481,IR_retnet#93999482,turnover#93999483] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string... ,None) (5) Scan csv Output [25]: [year#93999459, retIC#93999460, resretIC#93999461, numcos#93999462, numdates#93999463, annual_bmret#93999464, annual_ret#93999465, std_ret#93999466, Sharpe_ret#93999467, PctPos_ret#93999468, TR_ret#93999469, IR_ret#93999470, annual_resret#93999471, std_resret#93999472, Sharpe_resret#93999473, PctPos_resret#93999474, TR_resret#93999475, IR_resret#93999476, annual_retnet#93999477, std_retnet#93999478, Sharpe_retnet#93999479, PctPos_retnet#93999480, TR_retnet#93999481, IR_retnet#93999482, turnover#93999483] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/estimize_signal_history/estimizesignal_preearnings/stats_year.csv] ReadSchema: struct<year:string,retIC:string,resretIC:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string> (6) Project [codegen id : 1] Output [25]: [CASE WHEN ((year#93999459 = NA) OR (year#93999459 = null)) THEN null ELSE cast(year#93999459 as float) END AS year#93999692, CASE WHEN ((retIC#93999460 = NA) OR (retIC#93999460 = null)) THEN null ELSE cast(retIC#93999460 as float) END AS retIC#93999694, CASE WHEN ((resretIC#93999461 = NA) OR (resretIC#93999461 = null)) THEN null ELSE cast(resretIC#93999461 as float) END AS resretIC#93999696, CASE WHEN ((numcos#93999462 = NA) OR (numcos#93999462 = null)) THEN null ELSE cast(numcos#93999462 as float) END AS numcos#93999698, CASE WHEN ((numdates#93999463 = NA) OR (numdates#93999463 = null)) THEN null ELSE cast(numdates#93999463 as int) END AS numdates#93999700, CASE WHEN ((annual_bmret#93999464 = NA) OR (annual_bmret#93999464 = null)) THEN null ELSE cast(annual_bmret#93999464 as float) END AS annual_bmret#93999702, CASE WHEN ((annual_ret#93999465 = NA) OR (annual_ret#93999465 = null)) THEN null ELSE cast(annual_ret#93999465 as float) END AS annual_ret#93999704, CASE WHEN ((std_ret#93999466 = NA) OR (std_ret#93999466 = null)) THEN null ELSE cast(std_ret#93999466 as float) END AS std_ret#93999706, CASE WHEN ((Sharpe_ret#93999467 = NA) OR (Sharpe_ret#93999467 = null)) THEN null ELSE cast(Sharpe_ret#93999467 as float) END AS Sharpe_ret#93999708, CASE WHEN ((PctPos_ret#93999468 = NA) OR (PctPos_ret#93999468 = null)) THEN null ELSE cast(PctPos_ret#93999468 as float) END AS PctPos_ret#93999710, CASE WHEN ((TR_ret#93999469 = NA) OR (TR_ret#93999469 = null)) THEN null ELSE cast(TR_ret#93999469 as float) END AS TR_ret#93999712, CASE WHEN ((IR_ret#93999470 = NA) OR (IR_ret#93999470 = null)) THEN null ELSE cast(IR_ret#93999470 as float) END AS IR_ret#93999714, CASE WHEN ((annual_resret#93999471 = NA) OR (annual_resret#93999471 = null)) THEN null ELSE cast(annual_resret#93999471 as float) END AS annual_resret#93999716, CASE WHEN ((std_resret#93999472 = NA) OR (std_resret#93999472 = null)) THEN null ELSE cast(std_resret#93999472 as float) END AS std_resret#93999718, CASE WHEN ((Sharpe_resret#93999473 = NA) OR (Sharpe_resret#93999473 = null)) THEN null ELSE cast(Sharpe_resret#93999473 as float) END AS Sharpe_resret#93999720, CASE WHEN ((PctPos_resret#93999474 = NA) OR (PctPos_resret#93999474 = null)) THEN null ELSE cast(PctPos_resret#93999474 as float) END AS PctPos_resret#93999722, CASE WHEN ((TR_resret#93999475 = NA) OR (TR_resret#93999475 = null)) THEN null ELSE cast(TR_resret#93999475 as float) END AS TR_resret#93999724, CASE WHEN ((IR_resret#93999476 = NA) OR (IR_resret#93999476 = null)) THEN null ELSE cast(IR_resret#93999476 as float) END AS IR_resret#93999726, CASE WHEN ((annual_retnet#93999477 = NA) OR (annual_retnet#93999477 = null)) THEN null ELSE cast(annual_retnet#93999477 as float) END AS annual_retnet#93999728, CASE WHEN ((std_retnet#93999478 = NA) OR (std_retnet#93999478 = null)) THEN null ELSE cast(std_retnet#93999478 as float) END AS std_retnet#93999730, CASE WHEN ((Sharpe_retnet#93999479 = NA) OR (Sharpe_retnet#93999479 = null)) THEN null ELSE cast(Sharpe_retnet#93999479 as float) END AS Sharpe_retnet#93999732, CASE WHEN ((PctPos_retnet#93999480 = NA) OR (PctPos_retnet#93999480 = null)) THEN null ELSE cast(PctPos_retnet#93999480 as float) END AS PctPos_retnet#93999734, CASE WHEN ((TR_retnet#93999481 = NA) OR (TR_retnet#93999481 = null)) THEN null ELSE cast(TR_retnet#93999481 as float) END AS TR_retnet#93999736, CASE WHEN ((IR_retnet#93999482 = NA) OR (IR_retnet#93999482 = null)) THEN null ELSE cast(IR_retnet#93999482 as float) END AS IR_retnet#93999738, CASE WHEN ((turnover#93999483 = NA) OR (turnover#93999483 = null)) THEN null ELSE cast(turnover#93999483 as float) END AS turnover#93999740] Input [25]: [year#93999459, retIC#93999460, resretIC#93999461, numcos#93999462, numdates#93999463, annual_bmret#93999464, annual_ret#93999465, std_ret#93999466, Sharpe_ret#93999467, PctPos_ret#93999468, TR_ret#93999469, IR_ret#93999470, annual_resret#93999471, std_resret#93999472, Sharpe_resret#93999473, PctPos_resret#93999474, TR_resret#93999475, IR_resret#93999476, annual_retnet#93999477, std_retnet#93999478, Sharpe_retnet#93999479, PctPos_retnet#93999480, TR_retnet#93999481, IR_retnet#93999482, turnover#93999483] (7) ColumnarToRow [codegen id : 1] Input [2]: [turnover#93999740, year#93999692] (8) Filter [codegen id : 1] Input [2]: [turnover#93999740, year#93999692] Condition : isnotnull(turnover#93999740) (9) Project [codegen id : 1] Output [3]: [year#93999692, turnover#93999740, (1.0 / cast(turnover#93999740 as double)) AS days_hold#93999774] Input [2]: [turnover#93999740, year#93999692] (10) Exchange Input [3]: [year#93999692, turnover#93999740, days_hold#93999774] Arguments: rangepartitioning(year#93999692 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7505140] (11) Sort [codegen id : 2] Input [3]: [year#93999692, turnover#93999740, days_hold#93999774] Arguments: [year#93999692 ASC NULLS FIRST], true, 0 (12) ColumnarToRow [codegen id : 1] Input [3]: [year#93999692, turnover#93999740, days_hold#93999774] (13) CollectLimit Input [3]: [year#93999692, turnover#93999740, days_hold#93999774] Arguments: 10000