== Physical Plan ==
CollectLimit (9)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Sort (8)
+- Exchange (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [3]: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
Arguments: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
(2) InMemoryRelation
Arguments: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Sort [date#94020145 ASC NULLS FIRST], true, 0
+- Exchange rangepartitioning(date#94020145 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7506840]
+- InMemoryTableScan [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
+- InMemoryRelation [date#94020145, allret#94020146, allcum#94020147, drawdown_gross#94020148, drawdown_length_gross#94020149, allretnet#94020151, allcumnet#94020152, drawdown_net#94020175, drawdown_length_net#94020176, numcos#94020177, largecum#94020288, largecumnet#94020289, largeret#94020291, largeretnet#94020292, midcum#94020738, midcumnet#94020741, midret#94020743, midretnet#94020769, smallcum#94020772, smallcumnet#94020777, smallret#94020779, smallretnet#94020780], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN (date#94019687 = null) THEN null ELSE cast(date#94019687 as date) END AS date#94019796, CASE WHEN ((allret#94019688 = NA) OR (allret#94019688 = null)) THEN null ELSE cast(allret#94019688 as float) END AS allret#94019798, CASE WHEN ((allcum#94019689 = NA) OR (allcum#94019689 = null)) THEN null ELSE cast(allcum#94019689 as float) END AS allcum#94019800, CASE WHEN ((drawdown_gross#94019690 = NA) OR (drawdown_gross#94019690 = null)) THEN null ELSE cast(drawdown_gross#94019690 as float) END AS drawdown_gross#94019846, CASE WHEN ((drawdown_length_gross#94019691 = NA) OR (drawdown_length_gross#94019691 = null)) THEN null ELSE cast(drawdown_length_gross#94019691 as int) END AS drawdown_length_gross#94019849, CASE WHEN ((allretnet#94019692 = NA) OR (allretnet#94019692 = null)) THEN null ELSE cast(allretnet#94019692 as float) END AS allretnet#94019873, CASE WHEN ((allcumnet#94019693 = NA) OR (allcumnet#94019693 = null)) THEN null ELSE cast(allcumnet#94019693 as float) END AS allcumnet#94019874, CASE WHEN ((drawdown_net#94019694 = NA) OR (drawdown_net#94019694 = null)) THEN null ELSE cast(drawdown_net#94019694 as float) END AS drawdown_net#94019897, CASE WHEN ((drawdown_length_net#94019695 = NA) OR (drawdown_length_net#94019695 = null)) THEN null ELSE cast(drawdown_length_net#94019695 as int) END AS drawdown_length_net#94019898, CASE WHEN ((numcos#94019696 = NA) OR (numcos#94019696 = null)) THEN null ELSE cast(numcos#94019696 as float) END AS numcos#94019899, CASE WHEN ((largecum#94019697 = NA) OR (largecum#94019697 = null)) THEN null ELSE cast(largecum#94019697 as float) END AS largecum#94019906, CASE WHEN ((largecumnet#94019698 = NA) OR (largecumnet#94019698 = null)) THEN null ELSE cast(largecumnet#94019698 as float) END AS largecumnet#94019913, CASE WHEN ((largeret#94019699 = NA) OR (largeret#94019699 = null)) THEN null ELSE cast(largeret#94019699 as float) END AS largeret#94019919, CASE WHEN ((largeretnet#94019700 = NA) OR (largeretnet#94019700 = null)) THEN null ELSE cast(largeretnet#94019700 as float) END AS largeretnet#94019926, CASE WHEN ((midcum#94019701 = NA) OR (midcum#94019701 = null)) THEN null ELSE cast(midcum#94019701 as float) END AS midcum#94019935, CASE WHEN ((midcumnet#94019702 = NA) OR (midcumnet#94019702 = null)) THEN null ELSE cast(midcumnet#94019702 as float) END AS midcumnet#94019969, CASE WHEN ((midret#94019703 = NA) OR (midret#94019703 = null)) THEN null ELSE cast(midret#94019703 as float) END AS midret#94019979, CASE WHEN ((midretnet#94019704 = NA) OR (midretnet#94019704 = null)) THEN null ELSE cast(midretnet#94019704 as float) END AS midretnet#94019986, CASE WHEN ((smallcum#94019705 = NA) OR (smallcum#94019705 = null)) THEN null ELSE cast(smallcum#94019705 as float) END AS smallcum#94019988, CASE WHEN ((smallcumnet#94019706 = NA) OR (smallcumnet#94019706 = null)) THEN null ELSE cast(smallcumnet#94019706 as float) END AS smallcumnet#94019990, CASE WHEN ((smallret#94019707 = NA) OR (smallret#94019707 = null)) THEN null ELSE cast(smallret#94019707 as float) END AS smallret#94019991, CASE WHEN ((smallretnet#94019708 = NA) OR (smallretnet#94019708 = null)) THEN null ELSE cast(smallretnet#94019708 as float) END AS smallretnet#94019992]
+- FileScan csv [date#94019687,allret#94019688,allcum#94019689,drawdown_gross#94019690,drawdown_length_gross#94019691,allretnet#94019692,allcumnet#94019693,drawdown_net#94019694,drawdown_length_net#94019695,numcos#94019696,largecum#94019697,largecumnet#94019698,largeret#94019699,largeretnet#94019700,midcum#94019701,midcumnet#94019702,midret#94019703,midretnet#94019704,smallcum#94019705,smallcumnet#94019706,smallret#94019707,smallretnet#94019708] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/RE..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None), [date#94020145 ASC NULLS FIRST]
(3) InMemoryTableScan
Output [3]: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
Arguments: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
(4) InMemoryRelation
Arguments: [date#94020145, allret#94020146, allcum#94020147, drawdown_gross#94020148, drawdown_length_gross#94020149, allretnet#94020151, allcumnet#94020152, drawdown_net#94020175, drawdown_length_net#94020176, numcos#94020177, largecum#94020288, largecumnet#94020289, largeret#94020291, largeretnet#94020292, midcum#94020738, midcumnet#94020741, midret#94020743, midretnet#94020769, smallcum#94020772, smallcumnet#94020777, smallret#94020779, smallretnet#94020780], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94019687 = null) THEN null ELSE cast(date#94019687 as date) END AS date#94019796, CASE WHEN ((allret#94019688 = NA) OR (allret#94019688 = null)) THEN null ELSE cast(allret#94019688 as float) END AS allret#94019798, CASE WHEN ((allcum#94019689 = NA) OR (allcum#94019689 = null)) THEN null ELSE cast(allcum#94019689 as float) END AS allcum#94019800, CASE WHEN ((drawdown_gross#94019690 = NA) OR (drawdown_gross#94019690 = null)) THEN null ELSE cast(drawdown_gross#94019690 as float) END AS drawdown_gross#94019846, CASE WHEN ((drawdown_length_gross#94019691 = NA) OR (drawdown_length_gross#94019691 = null)) THEN null ELSE cast(drawdown_length_gross#94019691 as int) END AS drawdown_length_gross#94019849, CASE WHEN ((allretnet#94019692 = NA) OR (allretnet#94019692 = null)) THEN null ELSE cast(allretnet#94019692 as float) END AS allretnet#94019873, CASE WHEN ((allcumnet#94019693 = NA) OR (allcumnet#94019693 = null)) THEN null ELSE cast(allcumnet#94019693 as float) END AS allcumnet#94019874, CASE WHEN ((drawdown_net#94019694 = NA) OR (drawdown_net#94019694 = null)) THEN null ELSE cast(drawdown_net#94019694 as float) END AS drawdown_net#94019897, CASE WHEN ((drawdown_length_net#94019695 = NA) OR (drawdown_length_net#94019695 = null)) THEN null ELSE cast(drawdown_length_net#94019695 as int) END AS drawdown_length_net#94019898, CASE WHEN ((numcos#94019696 = NA) OR (numcos#94019696 = null)) THEN null ELSE cast(numcos#94019696 as float) END AS numcos#94019899, CASE WHEN ((largecum#94019697 = NA) OR (largecum#94019697 = null)) THEN null ELSE cast(largecum#94019697 as float) END AS largecum#94019906, CASE WHEN ((largecumnet#94019698 = NA) OR (largecumnet#94019698 = null)) THEN null ELSE cast(largecumnet#94019698 as float) END AS largecumnet#94019913, CASE WHEN ((largeret#94019699 = NA) OR (largeret#94019699 = null)) THEN null ELSE cast(largeret#94019699 as float) END AS largeret#94019919, CASE WHEN ((largeretnet#94019700 = NA) OR (largeretnet#94019700 = null)) THEN null ELSE cast(largeretnet#94019700 as float) END AS largeretnet#94019926, CASE WHEN ((midcum#94019701 = NA) OR (midcum#94019701 = null)) THEN null ELSE cast(midcum#94019701 as float) END AS midcum#94019935, CASE WHEN ((midcumnet#94019702 = NA) OR (midcumnet#94019702 = null)) THEN null ELSE cast(midcumnet#94019702 as float) END AS midcumnet#94019969, CASE WHEN ((midret#94019703 = NA) OR (midret#94019703 = null)) THEN null ELSE cast(midret#94019703 as float) END AS midret#94019979, CASE WHEN ((midretnet#94019704 = NA) OR (midretnet#94019704 = null)) THEN null ELSE cast(midretnet#94019704 as float) END AS midretnet#94019986, CASE WHEN ((smallcum#94019705 = NA) OR (smallcum#94019705 = null)) THEN null ELSE cast(smallcum#94019705 as float) END AS smallcum#94019988, CASE WHEN ((smallcumnet#94019706 = NA) OR (smallcumnet#94019706 = null)) THEN null ELSE cast(smallcumnet#94019706 as float) END AS smallcumnet#94019990, CASE WHEN ((smallret#94019707 = NA) OR (smallret#94019707 = null)) THEN null ELSE cast(smallret#94019707 as float) END AS smallret#94019991, CASE WHEN ((smallretnet#94019708 = NA) OR (smallretnet#94019708 = null)) THEN null ELSE cast(smallretnet#94019708 as float) END AS smallretnet#94019992]
+- FileScan csv [date#94019687,allret#94019688,allcum#94019689,drawdown_gross#94019690,drawdown_length_gross#94019691,allretnet#94019692,allcumnet#94019693,drawdown_net#94019694,drawdown_length_net#94019695,numcos#94019696,largecum#94019697,largecumnet#94019698,largeret#94019699,largeretnet#94019700,midcum#94019701,midcumnet#94019702,midret#94019703,midretnet#94019704,smallcum#94019705,smallcumnet#94019706,smallret#94019707,smallretnet#94019708] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/RE..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None)
(5) Scan csv
Output [22]: [date#94019687, allret#94019688, allcum#94019689, drawdown_gross#94019690, drawdown_length_gross#94019691, allretnet#94019692, allcumnet#94019693, drawdown_net#94019694, drawdown_length_net#94019695, numcos#94019696, largecum#94019697, largecumnet#94019698, largeret#94019699, largeretnet#94019700, midcum#94019701, midcumnet#94019702, midret#94019703, midretnet#94019704, smallcum#94019705, smallcumnet#94019706, smallret#94019707, smallretnet#94019708]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/REPORT_cumret.csv]
ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string>
(6) Project [codegen id : 1]
Output [22]: [CASE WHEN (date#94019687 = null) THEN null ELSE cast(date#94019687 as date) END AS date#94019796, CASE WHEN ((allret#94019688 = NA) OR (allret#94019688 = null)) THEN null ELSE cast(allret#94019688 as float) END AS allret#94019798, CASE WHEN ((allcum#94019689 = NA) OR (allcum#94019689 = null)) THEN null ELSE cast(allcum#94019689 as float) END AS allcum#94019800, CASE WHEN ((drawdown_gross#94019690 = NA) OR (drawdown_gross#94019690 = null)) THEN null ELSE cast(drawdown_gross#94019690 as float) END AS drawdown_gross#94019846, CASE WHEN ((drawdown_length_gross#94019691 = NA) OR (drawdown_length_gross#94019691 = null)) THEN null ELSE cast(drawdown_length_gross#94019691 as int) END AS drawdown_length_gross#94019849, CASE WHEN ((allretnet#94019692 = NA) OR (allretnet#94019692 = null)) THEN null ELSE cast(allretnet#94019692 as float) END AS allretnet#94019873, CASE WHEN ((allcumnet#94019693 = NA) OR (allcumnet#94019693 = null)) THEN null ELSE cast(allcumnet#94019693 as float) END AS allcumnet#94019874, CASE WHEN ((drawdown_net#94019694 = NA) OR (drawdown_net#94019694 = null)) THEN null ELSE cast(drawdown_net#94019694 as float) END AS drawdown_net#94019897, CASE WHEN ((drawdown_length_net#94019695 = NA) OR (drawdown_length_net#94019695 = null)) THEN null ELSE cast(drawdown_length_net#94019695 as int) END AS drawdown_length_net#94019898, CASE WHEN ((numcos#94019696 = NA) OR (numcos#94019696 = null)) THEN null ELSE cast(numcos#94019696 as float) END AS numcos#94019899, CASE WHEN ((largecum#94019697 = NA) OR (largecum#94019697 = null)) THEN null ELSE cast(largecum#94019697 as float) END AS largecum#94019906, CASE WHEN ((largecumnet#94019698 = NA) OR (largecumnet#94019698 = null)) THEN null ELSE cast(largecumnet#94019698 as float) END AS largecumnet#94019913, CASE WHEN ((largeret#94019699 = NA) OR (largeret#94019699 = null)) THEN null ELSE cast(largeret#94019699 as float) END AS largeret#94019919, CASE WHEN ((largeretnet#94019700 = NA) OR (largeretnet#94019700 = null)) THEN null ELSE cast(largeretnet#94019700 as float) END AS largeretnet#94019926, CASE WHEN ((midcum#94019701 = NA) OR (midcum#94019701 = null)) THEN null ELSE cast(midcum#94019701 as float) END AS midcum#94019935, CASE WHEN ((midcumnet#94019702 = NA) OR (midcumnet#94019702 = null)) THEN null ELSE cast(midcumnet#94019702 as float) END AS midcumnet#94019969, CASE WHEN ((midret#94019703 = NA) OR (midret#94019703 = null)) THEN null ELSE cast(midret#94019703 as float) END AS midret#94019979, CASE WHEN ((midretnet#94019704 = NA) OR (midretnet#94019704 = null)) THEN null ELSE cast(midretnet#94019704 as float) END AS midretnet#94019986, CASE WHEN ((smallcum#94019705 = NA) OR (smallcum#94019705 = null)) THEN null ELSE cast(smallcum#94019705 as float) END AS smallcum#94019988, CASE WHEN ((smallcumnet#94019706 = NA) OR (smallcumnet#94019706 = null)) THEN null ELSE cast(smallcumnet#94019706 as float) END AS smallcumnet#94019990, CASE WHEN ((smallret#94019707 = NA) OR (smallret#94019707 = null)) THEN null ELSE cast(smallret#94019707 as float) END AS smallret#94019991, CASE WHEN ((smallretnet#94019708 = NA) OR (smallretnet#94019708 = null)) THEN null ELSE cast(smallretnet#94019708 as float) END AS smallretnet#94019992]
Input [22]: [date#94019687, allret#94019688, allcum#94019689, drawdown_gross#94019690, drawdown_length_gross#94019691, allretnet#94019692, allcumnet#94019693, drawdown_net#94019694, drawdown_length_net#94019695, numcos#94019696, largecum#94019697, largecumnet#94019698, largeret#94019699, largeretnet#94019700, midcum#94019701, midcumnet#94019702, midret#94019703, midretnet#94019704, smallcum#94019705, smallcumnet#94019706, smallret#94019707, smallretnet#94019708]
(7) Exchange
Input [3]: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
Arguments: rangepartitioning(date#94020145 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7506840]
(8) Sort [codegen id : 1]
Input [3]: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
Arguments: [date#94020145 ASC NULLS FIRST], true, 0
(9) CollectLimit
Input [3]: [date#94020145, drawdown_gross#94020148, drawdown_net#94020175]
Arguments: 1000000