== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [date#94149970, daily_gross#94150155, daily_net#94150156] Arguments: [date#94149970, daily_gross#94150155, daily_net#94150156] (2) InMemoryRelation Arguments: [date#94149970, daily_gross#94150155, daily_net#94150156], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94149970 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94149970 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7517155] +- *(1) Project [date#94149970, allret#94149979 AS daily_gross#94150155, allretnet#94149990 AS daily_net#94150156] +- InMemoryTableScan [allret#94149979, allretnet#94149990, date#94149970] +- InMemoryRelation [date#94149970, allret#94149979, allcum#94149982, drawdown_gross#94149985, drawdown_length_gross#94149988, allretnet#94149990, allcumnet#94149991, drawdown_net#94149992, drawdown_length_net#94149993, numcos#94150000, largecum#94150002, largecumnet#94150008, largeret#94150009, largeretnet#94150010, midcum#94150011, midcumnet#94150012, midret#94150013, midretnet#94150064, smallcum#94150065, smallcumnet#94150066, smallret#94150067, smallretnet#94150068], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94149758 = null) THEN null ELSE cast(date#94149758 as date) END AS date#94149970, CASE WHEN ((allret#94149759 = NA) OR (allret#94149759 = null)) THEN null ELSE cast(allret#94149759 as float) END AS allret#94149979, CASE WHEN ((allcum#94149760 = NA) OR (allcum#94149760 = null)) THEN null ELSE cast(allcum#94149760 as float) END AS allcum#94149982, CASE WHEN ((drawdown_gross#94149761 = NA) OR (drawdown_gross#94149761 = null)) THEN null ELSE cast(drawdown_gross#94149761 as float) END AS drawdown_gross#94149985, CASE WHEN ((drawdown_length_gross#94149762 = NA) OR (drawdown_length_gross#94149762 = null)) THEN null ELSE cast(drawdown_length_gross#94149762 as int) END AS drawdown_length_gross#94149988, CASE WHEN ((allretnet#94149763 = NA) OR (allretnet#94149763 = null)) THEN null ELSE cast(allretnet#94149763 as float) END AS allretnet#94149990, CASE WHEN ((allcumnet#94149764 = NA) OR (allcumnet#94149764 = null)) THEN null ELSE cast(allcumnet#94149764 as float) END AS allcumnet#94149991, CASE WHEN ((drawdown_net#94149765 = NA) OR (drawdown_net#94149765 = null)) THEN null ELSE cast(drawdown_net#94149765 as float) END AS drawdown_net#94149992, CASE WHEN ((drawdown_length_net#94149766 = NA) OR (drawdown_length_net#94149766 = null)) THEN null ELSE cast(drawdown_length_net#94149766 as int) END AS drawdown_length_net#94149993, CASE WHEN ((numcos#94149767 = NA) OR (numcos#94149767 = null)) THEN null ELSE cast(numcos#94149767 as float) END AS numcos#94150000, CASE WHEN ((largecum#94149768 = NA) OR (largecum#94149768 = null)) THEN null ELSE cast(largecum#94149768 as float) END AS largecum#94150002, CASE WHEN ((largecumnet#94149769 = NA) OR (largecumnet#94149769 = null)) THEN null ELSE cast(largecumnet#94149769 as float) END AS largecumnet#94150008, CASE WHEN ((largeret#94149770 = NA) OR (largeret#94149770 = null)) THEN null ELSE cast(largeret#94149770 as float) END AS largeret#94150009, CASE WHEN ((largeretnet#94149771 = NA) OR (largeretnet#94149771 = null)) THEN null ELSE cast(largeretnet#94149771 as float) END AS largeretnet#94150010, CASE WHEN ((midcum#94149772 = NA) OR (midcum#94149772 = null)) THEN null ELSE cast(midcum#94149772 as float) END AS midcum#94150011, CASE WHEN ((midcumnet#94149773 = NA) OR (midcumnet#94149773 = null)) THEN null ELSE cast(midcumnet#94149773 as float) END AS midcumnet#94150012, CASE WHEN ((midret#94149774 = NA) OR (midret#94149774 = null)) THEN null ELSE cast(midret#94149774 as float) END AS midret#94150013, CASE WHEN ((midretnet#94149775 = NA) OR (midretnet#94149775 = null)) THEN null ELSE cast(midretnet#94149775 as float) END AS midretnet#94150064, CASE WHEN ((smallcum#94149776 = NA) OR (smallcum#94149776 = null)) THEN null ELSE cast(smallcum#94149776 as float) END AS smallcum#94150065, CASE WHEN ((smallcumnet#94149777 = NA) OR (smallcumnet#94149777 = null)) THEN null ELSE cast(smallcumnet#94149777 as float) END AS smallcumnet#94150066, CASE WHEN ((smallret#94149778 = NA) OR (smallret#94149778 = null)) THEN null ELSE cast(smallret#94149778 as float) END AS smallret#94150067, CASE WHEN ((smallretnet#94149779 = NA) OR (smallretnet#94149779 = null)) THEN null ELSE cast(smallretnet#94149779 as float) END AS smallretnet#94150068] +- FileScan csv [date#94149758,allret#94149759,allcum#94149760,drawdown_gross#94149761,drawdown_length_gross#94149762,allretnet#94149763,allcumnet#94149764,drawdown_net#94149765,drawdown_length_net#94149766,numcos#94149767,largecum#94149768,largecumnet#94149769,largeret#94149770,largeretnet#94149771,midcum#94149772,midcumnet#94149773,midret#94149774,midretnet#94149775,smallcum#94149776,smallcumnet#94149777,smallret#94149778,smallretnet#94149779] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/size/REPO..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94149970 ASC NULLS FIRST] (3) InMemoryTableScan Output [3]: [allret#94149979, allretnet#94149990, date#94149970] Arguments: [allret#94149979, allretnet#94149990, date#94149970] (4) InMemoryRelation Arguments: [date#94149970, allret#94149979, allcum#94149982, drawdown_gross#94149985, drawdown_length_gross#94149988, allretnet#94149990, allcumnet#94149991, drawdown_net#94149992, drawdown_length_net#94149993, numcos#94150000, largecum#94150002, largecumnet#94150008, largeret#94150009, largeretnet#94150010, midcum#94150011, midcumnet#94150012, midret#94150013, midretnet#94150064, smallcum#94150065, smallcumnet#94150066, smallret#94150067, smallretnet#94150068], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94149758 = null) THEN null ELSE cast(date#94149758 as date) END AS date#94149970, CASE WHEN ((allret#94149759 = NA) OR (allret#94149759 = null)) THEN null ELSE cast(allret#94149759 as float) END AS allret#94149979, CASE WHEN ((allcum#94149760 = NA) OR (allcum#94149760 = null)) THEN null ELSE cast(allcum#94149760 as float) END AS allcum#94149982, CASE WHEN ((drawdown_gross#94149761 = NA) OR (drawdown_gross#94149761 = null)) THEN null ELSE cast(drawdown_gross#94149761 as float) END AS drawdown_gross#94149985, CASE WHEN ((drawdown_length_gross#94149762 = NA) OR (drawdown_length_gross#94149762 = null)) THEN null ELSE cast(drawdown_length_gross#94149762 as int) END AS drawdown_length_gross#94149988, CASE WHEN ((allretnet#94149763 = NA) OR (allretnet#94149763 = null)) THEN null ELSE cast(allretnet#94149763 as float) END AS allretnet#94149990, CASE WHEN ((allcumnet#94149764 = NA) OR (allcumnet#94149764 = null)) THEN null ELSE cast(allcumnet#94149764 as float) END AS allcumnet#94149991, CASE WHEN ((drawdown_net#94149765 = NA) OR (drawdown_net#94149765 = null)) THEN null ELSE cast(drawdown_net#94149765 as float) END AS drawdown_net#94149992, CASE WHEN ((drawdown_length_net#94149766 = NA) OR (drawdown_length_net#94149766 = null)) THEN null ELSE cast(drawdown_length_net#94149766 as int) END AS drawdown_length_net#94149993, CASE WHEN ((numcos#94149767 = NA) OR (numcos#94149767 = null)) THEN null ELSE cast(numcos#94149767 as float) END AS numcos#94150000, CASE WHEN ((largecum#94149768 = NA) OR (largecum#94149768 = null)) THEN null ELSE cast(largecum#94149768 as float) END AS largecum#94150002, CASE WHEN ((largecumnet#94149769 = NA) OR (largecumnet#94149769 = null)) THEN null ELSE cast(largecumnet#94149769 as float) END AS largecumnet#94150008, CASE WHEN ((largeret#94149770 = NA) OR (largeret#94149770 = null)) THEN null ELSE cast(largeret#94149770 as float) END AS largeret#94150009, CASE WHEN ((largeretnet#94149771 = NA) OR (largeretnet#94149771 = null)) THEN null ELSE cast(largeretnet#94149771 as float) END AS largeretnet#94150010, CASE WHEN ((midcum#94149772 = NA) OR (midcum#94149772 = null)) THEN null ELSE cast(midcum#94149772 as float) END AS midcum#94150011, CASE WHEN ((midcumnet#94149773 = NA) OR (midcumnet#94149773 = null)) THEN null ELSE cast(midcumnet#94149773 as float) END AS midcumnet#94150012, CASE WHEN ((midret#94149774 = NA) OR (midret#94149774 = null)) THEN null ELSE cast(midret#94149774 as float) END AS midret#94150013, CASE WHEN ((midretnet#94149775 = NA) OR (midretnet#94149775 = null)) THEN null ELSE cast(midretnet#94149775 as float) END AS midretnet#94150064, CASE WHEN ((smallcum#94149776 = NA) OR (smallcum#94149776 = null)) THEN null ELSE cast(smallcum#94149776 as float) END AS smallcum#94150065, CASE WHEN ((smallcumnet#94149777 = NA) OR (smallcumnet#94149777 = null)) THEN null ELSE cast(smallcumnet#94149777 as float) END AS smallcumnet#94150066, CASE WHEN ((smallret#94149778 = NA) OR (smallret#94149778 = null)) THEN null ELSE cast(smallret#94149778 as float) END AS smallret#94150067, CASE WHEN ((smallretnet#94149779 = NA) OR (smallretnet#94149779 = null)) THEN null ELSE cast(smallretnet#94149779 as float) END AS smallretnet#94150068] +- FileScan csv [date#94149758,allret#94149759,allcum#94149760,drawdown_gross#94149761,drawdown_length_gross#94149762,allretnet#94149763,allcumnet#94149764,drawdown_net#94149765,drawdown_length_net#94149766,numcos#94149767,largecum#94149768,largecumnet#94149769,largeret#94149770,largeretnet#94149771,midcum#94149772,midcumnet#94149773,midret#94149774,midretnet#94149775,smallcum#94149776,smallcumnet#94149777,smallret#94149778,smallretnet#94149779] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/size/REPO..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94149758, allret#94149759, allcum#94149760, drawdown_gross#94149761, drawdown_length_gross#94149762, allretnet#94149763, allcumnet#94149764, drawdown_net#94149765, drawdown_length_net#94149766, numcos#94149767, largecum#94149768, largecumnet#94149769, largeret#94149770, largeretnet#94149771, midcum#94149772, midcumnet#94149773, midret#94149774, midretnet#94149775, smallcum#94149776, smallcumnet#94149777, smallret#94149778, smallretnet#94149779] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/size/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94149758 = null) THEN null ELSE cast(date#94149758 as date) END AS date#94149970, CASE WHEN ((allret#94149759 = NA) OR (allret#94149759 = null)) THEN null ELSE cast(allret#94149759 as float) END AS allret#94149979, CASE WHEN ((allcum#94149760 = NA) OR (allcum#94149760 = null)) THEN null ELSE cast(allcum#94149760 as float) END AS allcum#94149982, CASE WHEN ((drawdown_gross#94149761 = NA) OR (drawdown_gross#94149761 = null)) THEN null ELSE cast(drawdown_gross#94149761 as float) END AS drawdown_gross#94149985, CASE WHEN ((drawdown_length_gross#94149762 = NA) OR (drawdown_length_gross#94149762 = null)) THEN null ELSE cast(drawdown_length_gross#94149762 as int) END AS drawdown_length_gross#94149988, CASE WHEN ((allretnet#94149763 = NA) OR (allretnet#94149763 = null)) THEN null ELSE cast(allretnet#94149763 as float) END AS allretnet#94149990, CASE WHEN ((allcumnet#94149764 = NA) OR (allcumnet#94149764 = null)) THEN null ELSE cast(allcumnet#94149764 as float) END AS allcumnet#94149991, CASE WHEN ((drawdown_net#94149765 = NA) OR (drawdown_net#94149765 = null)) THEN null ELSE cast(drawdown_net#94149765 as float) END AS drawdown_net#94149992, CASE WHEN ((drawdown_length_net#94149766 = NA) OR (drawdown_length_net#94149766 = null)) THEN null ELSE cast(drawdown_length_net#94149766 as int) END AS drawdown_length_net#94149993, CASE WHEN ((numcos#94149767 = NA) OR (numcos#94149767 = null)) THEN null ELSE cast(numcos#94149767 as float) END AS numcos#94150000, CASE WHEN ((largecum#94149768 = NA) OR (largecum#94149768 = null)) THEN null ELSE cast(largecum#94149768 as float) END AS largecum#94150002, CASE WHEN ((largecumnet#94149769 = NA) OR (largecumnet#94149769 = null)) THEN null ELSE cast(largecumnet#94149769 as float) END AS largecumnet#94150008, CASE WHEN ((largeret#94149770 = NA) OR (largeret#94149770 = null)) THEN null ELSE cast(largeret#94149770 as float) END AS largeret#94150009, CASE WHEN ((largeretnet#94149771 = NA) OR (largeretnet#94149771 = null)) THEN null ELSE cast(largeretnet#94149771 as float) END AS largeretnet#94150010, CASE WHEN ((midcum#94149772 = NA) OR (midcum#94149772 = null)) THEN null ELSE cast(midcum#94149772 as float) END AS midcum#94150011, CASE WHEN ((midcumnet#94149773 = NA) OR (midcumnet#94149773 = null)) THEN null ELSE cast(midcumnet#94149773 as float) END AS midcumnet#94150012, CASE WHEN ((midret#94149774 = NA) OR (midret#94149774 = null)) THEN null ELSE cast(midret#94149774 as float) END AS midret#94150013, CASE WHEN ((midretnet#94149775 = NA) OR (midretnet#94149775 = null)) THEN null ELSE cast(midretnet#94149775 as float) END AS midretnet#94150064, CASE WHEN ((smallcum#94149776 = NA) OR (smallcum#94149776 = null)) THEN null ELSE cast(smallcum#94149776 as float) END AS smallcum#94150065, CASE WHEN ((smallcumnet#94149777 = NA) OR (smallcumnet#94149777 = null)) THEN null ELSE cast(smallcumnet#94149777 as float) END AS smallcumnet#94150066, CASE WHEN ((smallret#94149778 = NA) OR (smallret#94149778 = null)) THEN null ELSE cast(smallret#94149778 as float) END AS smallret#94150067, CASE WHEN ((smallretnet#94149779 = NA) OR (smallretnet#94149779 = null)) THEN null ELSE cast(smallretnet#94149779 as float) END AS smallretnet#94150068] Input [22]: [date#94149758, allret#94149759, allcum#94149760, drawdown_gross#94149761, drawdown_length_gross#94149762, allretnet#94149763, allcumnet#94149764, drawdown_net#94149765, drawdown_length_net#94149766, numcos#94149767, largecum#94149768, largecumnet#94149769, largeret#94149770, largeretnet#94149771, midcum#94149772, midcumnet#94149773, midret#94149774, midretnet#94149775, smallcum#94149776, smallcumnet#94149777, smallret#94149778, smallretnet#94149779] (7) Project [codegen id : 1] Output [3]: [date#94149970, allret#94149979 AS daily_gross#94150155, allretnet#94149990 AS daily_net#94150156] Input [3]: [allret#94149979, allretnet#94149990, date#94149970] (8) Exchange Input [3]: [date#94149970, daily_gross#94150155, daily_net#94150156] Arguments: rangepartitioning(date#94149970 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7517155] (9) Sort [codegen id : 2] Input [3]: [date#94149970, daily_gross#94150155, daily_net#94150156] Arguments: [date#94149970 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [3]: [date#94149970, daily_gross#94150155, daily_net#94150156] Arguments: 1000000