== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [2]: [date#94019796, drawdown_length_gross#94020028] Arguments: [date#94019796, drawdown_length_gross#94020028] (2) InMemoryRelation Arguments: [date#94019796, drawdown_length_gross#94020028], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94019796 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94019796 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7506862] +- *(1) Project [date#94019796, (drawdown_length_gross#94019849 * -1) AS drawdown_length_gross#94020028] +- InMemoryTableScan [date#94019796, drawdown_length_gross#94019849] +- InMemoryRelation [date#94019796, allret#94019798, allcum#94019800, drawdown_gross#94019846, drawdown_length_gross#94019849, allretnet#94019873, allcumnet#94019874, drawdown_net#94019897, drawdown_length_net#94019898, numcos#94019899, largecum#94019906, largecumnet#94019913, largeret#94019919, largeretnet#94019926, midcum#94019935, midcumnet#94019969, midret#94019979, midretnet#94019986, smallcum#94019988, smallcumnet#94019990, smallret#94019991, smallretnet#94019992], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94019687 = null) THEN null ELSE cast(date#94019687 as date) END AS date#94019796, CASE WHEN ((allret#94019688 = NA) OR (allret#94019688 = null)) THEN null ELSE cast(allret#94019688 as float) END AS allret#94019798, CASE WHEN ((allcum#94019689 = NA) OR (allcum#94019689 = null)) THEN null ELSE cast(allcum#94019689 as float) END AS allcum#94019800, CASE WHEN ((drawdown_gross#94019690 = NA) OR (drawdown_gross#94019690 = null)) THEN null ELSE cast(drawdown_gross#94019690 as float) END AS drawdown_gross#94019846, CASE WHEN ((drawdown_length_gross#94019691 = NA) OR (drawdown_length_gross#94019691 = null)) THEN null ELSE cast(drawdown_length_gross#94019691 as int) END AS drawdown_length_gross#94019849, CASE WHEN ((allretnet#94019692 = NA) OR (allretnet#94019692 = null)) THEN null ELSE cast(allretnet#94019692 as float) END AS allretnet#94019873, CASE WHEN ((allcumnet#94019693 = NA) OR (allcumnet#94019693 = null)) THEN null ELSE cast(allcumnet#94019693 as float) END AS allcumnet#94019874, CASE WHEN ((drawdown_net#94019694 = NA) OR (drawdown_net#94019694 = null)) THEN null ELSE cast(drawdown_net#94019694 as float) END AS drawdown_net#94019897, CASE WHEN ((drawdown_length_net#94019695 = NA) OR (drawdown_length_net#94019695 = null)) THEN null ELSE cast(drawdown_length_net#94019695 as int) END AS drawdown_length_net#94019898, CASE WHEN ((numcos#94019696 = NA) OR (numcos#94019696 = null)) THEN null ELSE cast(numcos#94019696 as float) END AS numcos#94019899, CASE WHEN ((largecum#94019697 = NA) OR (largecum#94019697 = null)) THEN null ELSE cast(largecum#94019697 as float) END AS largecum#94019906, CASE WHEN ((largecumnet#94019698 = NA) OR (largecumnet#94019698 = null)) THEN null ELSE cast(largecumnet#94019698 as float) END AS largecumnet#94019913, CASE WHEN ((largeret#94019699 = NA) OR (largeret#94019699 = null)) THEN null ELSE cast(largeret#94019699 as float) END AS largeret#94019919, CASE WHEN ((largeretnet#94019700 = NA) OR (largeretnet#94019700 = null)) THEN null ELSE cast(largeretnet#94019700 as float) END AS largeretnet#94019926, CASE WHEN ((midcum#94019701 = NA) OR (midcum#94019701 = null)) THEN null ELSE cast(midcum#94019701 as float) END AS midcum#94019935, CASE WHEN ((midcumnet#94019702 = NA) OR (midcumnet#94019702 = null)) THEN null ELSE cast(midcumnet#94019702 as float) END AS midcumnet#94019969, CASE WHEN ((midret#94019703 = NA) OR (midret#94019703 = null)) THEN null ELSE cast(midret#94019703 as float) END AS midret#94019979, CASE WHEN ((midretnet#94019704 = NA) OR (midretnet#94019704 = null)) THEN null ELSE cast(midretnet#94019704 as float) END AS midretnet#94019986, CASE WHEN ((smallcum#94019705 = NA) OR (smallcum#94019705 = null)) THEN null ELSE cast(smallcum#94019705 as float) END AS smallcum#94019988, CASE WHEN ((smallcumnet#94019706 = NA) OR (smallcumnet#94019706 = null)) THEN null ELSE cast(smallcumnet#94019706 as float) END AS smallcumnet#94019990, CASE WHEN ((smallret#94019707 = NA) OR (smallret#94019707 = null)) THEN null ELSE cast(smallret#94019707 as float) END AS smallret#94019991, CASE WHEN ((smallretnet#94019708 = NA) OR (smallretnet#94019708 = null)) THEN null ELSE cast(smallretnet#94019708 as float) END AS smallretnet#94019992] +- FileScan csv [date#94019687,allret#94019688,allcum#94019689,drawdown_gross#94019690,drawdown_length_gross#94019691,allretnet#94019692,allcumnet#94019693,drawdown_net#94019694,drawdown_length_net#94019695,numcos#94019696,largecum#94019697,largecumnet#94019698,largeret#94019699,largeretnet#94019700,midcum#94019701,midcumnet#94019702,midret#94019703,midretnet#94019704,smallcum#94019705,smallcumnet#94019706,smallret#94019707,smallretnet#94019708] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/RE..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94019796 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [date#94019796, drawdown_length_gross#94019849] Arguments: [date#94019796, drawdown_length_gross#94019849] (4) InMemoryRelation Arguments: [date#94019796, allret#94019798, allcum#94019800, drawdown_gross#94019846, drawdown_length_gross#94019849, allretnet#94019873, allcumnet#94019874, drawdown_net#94019897, drawdown_length_net#94019898, numcos#94019899, largecum#94019906, largecumnet#94019913, largeret#94019919, largeretnet#94019926, midcum#94019935, midcumnet#94019969, midret#94019979, midretnet#94019986, smallcum#94019988, smallcumnet#94019990, smallret#94019991, smallretnet#94019992], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94019687 = null) THEN null ELSE cast(date#94019687 as date) END AS date#94019796, CASE WHEN ((allret#94019688 = NA) OR (allret#94019688 = null)) THEN null ELSE cast(allret#94019688 as float) END AS allret#94019798, CASE WHEN ((allcum#94019689 = NA) OR (allcum#94019689 = null)) THEN null ELSE cast(allcum#94019689 as float) END AS allcum#94019800, CASE WHEN ((drawdown_gross#94019690 = NA) OR (drawdown_gross#94019690 = null)) THEN null ELSE cast(drawdown_gross#94019690 as float) END AS drawdown_gross#94019846, CASE WHEN ((drawdown_length_gross#94019691 = NA) OR (drawdown_length_gross#94019691 = null)) THEN null ELSE cast(drawdown_length_gross#94019691 as int) END AS drawdown_length_gross#94019849, CASE WHEN ((allretnet#94019692 = NA) OR (allretnet#94019692 = null)) THEN null ELSE cast(allretnet#94019692 as float) END AS allretnet#94019873, CASE WHEN ((allcumnet#94019693 = NA) OR (allcumnet#94019693 = null)) THEN null ELSE cast(allcumnet#94019693 as float) END AS allcumnet#94019874, CASE WHEN ((drawdown_net#94019694 = NA) OR (drawdown_net#94019694 = null)) THEN null ELSE cast(drawdown_net#94019694 as float) END AS drawdown_net#94019897, CASE WHEN ((drawdown_length_net#94019695 = NA) OR (drawdown_length_net#94019695 = null)) THEN null ELSE cast(drawdown_length_net#94019695 as int) END AS drawdown_length_net#94019898, CASE WHEN ((numcos#94019696 = NA) OR (numcos#94019696 = null)) THEN null ELSE cast(numcos#94019696 as float) END AS numcos#94019899, CASE WHEN ((largecum#94019697 = NA) OR (largecum#94019697 = null)) THEN null ELSE cast(largecum#94019697 as float) END AS largecum#94019906, CASE WHEN ((largecumnet#94019698 = NA) OR (largecumnet#94019698 = null)) THEN null ELSE cast(largecumnet#94019698 as float) END AS largecumnet#94019913, CASE WHEN ((largeret#94019699 = NA) OR (largeret#94019699 = null)) THEN null ELSE cast(largeret#94019699 as float) END AS largeret#94019919, CASE WHEN ((largeretnet#94019700 = NA) OR (largeretnet#94019700 = null)) THEN null ELSE cast(largeretnet#94019700 as float) END AS largeretnet#94019926, CASE WHEN ((midcum#94019701 = NA) OR (midcum#94019701 = null)) THEN null ELSE cast(midcum#94019701 as float) END AS midcum#94019935, CASE WHEN ((midcumnet#94019702 = NA) OR (midcumnet#94019702 = null)) THEN null ELSE cast(midcumnet#94019702 as float) END AS midcumnet#94019969, CASE WHEN ((midret#94019703 = NA) OR (midret#94019703 = null)) THEN null ELSE cast(midret#94019703 as float) END AS midret#94019979, CASE WHEN ((midretnet#94019704 = NA) OR (midretnet#94019704 = null)) THEN null ELSE cast(midretnet#94019704 as float) END AS midretnet#94019986, CASE WHEN ((smallcum#94019705 = NA) OR (smallcum#94019705 = null)) THEN null ELSE cast(smallcum#94019705 as float) END AS smallcum#94019988, CASE WHEN ((smallcumnet#94019706 = NA) OR (smallcumnet#94019706 = null)) THEN null ELSE cast(smallcumnet#94019706 as float) END AS smallcumnet#94019990, CASE WHEN ((smallret#94019707 = NA) OR (smallret#94019707 = null)) THEN null ELSE cast(smallret#94019707 as float) END AS smallret#94019991, CASE WHEN ((smallretnet#94019708 = NA) OR (smallretnet#94019708 = null)) THEN null ELSE cast(smallretnet#94019708 as float) END AS smallretnet#94019992] +- FileScan csv [date#94019687,allret#94019688,allcum#94019689,drawdown_gross#94019690,drawdown_length_gross#94019691,allretnet#94019692,allcumnet#94019693,drawdown_net#94019694,drawdown_length_net#94019695,numcos#94019696,largecum#94019697,largecumnet#94019698,largeret#94019699,largeretnet#94019700,midcum#94019701,midcumnet#94019702,midret#94019703,midretnet#94019704,smallcum#94019705,smallcumnet#94019706,smallret#94019707,smallretnet#94019708] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/RE..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94019687, allret#94019688, allcum#94019689, drawdown_gross#94019690, drawdown_length_gross#94019691, allretnet#94019692, allcumnet#94019693, drawdown_net#94019694, drawdown_length_net#94019695, numcos#94019696, largecum#94019697, largecumnet#94019698, largeret#94019699, largeretnet#94019700, midcum#94019701, midcumnet#94019702, midret#94019703, midretnet#94019704, smallcum#94019705, smallcumnet#94019706, smallret#94019707, smallretnet#94019708] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94019687 = null) THEN null ELSE cast(date#94019687 as date) END AS date#94019796, CASE WHEN ((allret#94019688 = NA) OR (allret#94019688 = null)) THEN null ELSE cast(allret#94019688 as float) END AS allret#94019798, CASE WHEN ((allcum#94019689 = NA) OR (allcum#94019689 = null)) THEN null ELSE cast(allcum#94019689 as float) END AS allcum#94019800, CASE WHEN ((drawdown_gross#94019690 = NA) OR (drawdown_gross#94019690 = null)) THEN null ELSE cast(drawdown_gross#94019690 as float) END AS drawdown_gross#94019846, CASE WHEN ((drawdown_length_gross#94019691 = NA) OR (drawdown_length_gross#94019691 = null)) THEN null ELSE cast(drawdown_length_gross#94019691 as int) END AS drawdown_length_gross#94019849, CASE WHEN ((allretnet#94019692 = NA) OR (allretnet#94019692 = null)) THEN null ELSE cast(allretnet#94019692 as float) END AS allretnet#94019873, CASE WHEN ((allcumnet#94019693 = NA) OR (allcumnet#94019693 = null)) THEN null ELSE cast(allcumnet#94019693 as float) END AS allcumnet#94019874, CASE WHEN ((drawdown_net#94019694 = NA) OR (drawdown_net#94019694 = null)) THEN null ELSE cast(drawdown_net#94019694 as float) END AS drawdown_net#94019897, CASE WHEN ((drawdown_length_net#94019695 = NA) OR (drawdown_length_net#94019695 = null)) THEN null ELSE cast(drawdown_length_net#94019695 as int) END AS drawdown_length_net#94019898, CASE WHEN ((numcos#94019696 = NA) OR (numcos#94019696 = null)) THEN null ELSE cast(numcos#94019696 as float) END AS numcos#94019899, CASE WHEN ((largecum#94019697 = NA) OR (largecum#94019697 = null)) THEN null ELSE cast(largecum#94019697 as float) END AS largecum#94019906, CASE WHEN ((largecumnet#94019698 = NA) OR (largecumnet#94019698 = null)) THEN null ELSE cast(largecumnet#94019698 as float) END AS largecumnet#94019913, CASE WHEN ((largeret#94019699 = NA) OR (largeret#94019699 = null)) THEN null ELSE cast(largeret#94019699 as float) END AS largeret#94019919, CASE WHEN ((largeretnet#94019700 = NA) OR (largeretnet#94019700 = null)) THEN null ELSE cast(largeretnet#94019700 as float) END AS largeretnet#94019926, CASE WHEN ((midcum#94019701 = NA) OR (midcum#94019701 = null)) THEN null ELSE cast(midcum#94019701 as float) END AS midcum#94019935, CASE WHEN ((midcumnet#94019702 = NA) OR (midcumnet#94019702 = null)) THEN null ELSE cast(midcumnet#94019702 as float) END AS midcumnet#94019969, CASE WHEN ((midret#94019703 = NA) OR (midret#94019703 = null)) THEN null ELSE cast(midret#94019703 as float) END AS midret#94019979, CASE WHEN ((midretnet#94019704 = NA) OR (midretnet#94019704 = null)) THEN null ELSE cast(midretnet#94019704 as float) END AS midretnet#94019986, CASE WHEN ((smallcum#94019705 = NA) OR (smallcum#94019705 = null)) THEN null ELSE cast(smallcum#94019705 as float) END AS smallcum#94019988, CASE WHEN ((smallcumnet#94019706 = NA) OR (smallcumnet#94019706 = null)) THEN null ELSE cast(smallcumnet#94019706 as float) END AS smallcumnet#94019990, CASE WHEN ((smallret#94019707 = NA) OR (smallret#94019707 = null)) THEN null ELSE cast(smallret#94019707 as float) END AS smallret#94019991, CASE WHEN ((smallretnet#94019708 = NA) OR (smallretnet#94019708 = null)) THEN null ELSE cast(smallretnet#94019708 as float) END AS smallretnet#94019992] Input [22]: [date#94019687, allret#94019688, allcum#94019689, drawdown_gross#94019690, drawdown_length_gross#94019691, allretnet#94019692, allcumnet#94019693, drawdown_net#94019694, drawdown_length_net#94019695, numcos#94019696, largecum#94019697, largecumnet#94019698, largeret#94019699, largeretnet#94019700, midcum#94019701, midcumnet#94019702, midret#94019703, midretnet#94019704, smallcum#94019705, smallcumnet#94019706, smallret#94019707, smallretnet#94019708] (7) Project [codegen id : 1] Output [2]: [date#94019796, (drawdown_length_gross#94019849 * -1) AS drawdown_length_gross#94020028] Input [2]: [date#94019796, drawdown_length_gross#94019849] (8) Exchange Input [2]: [date#94019796, drawdown_length_gross#94020028] Arguments: rangepartitioning(date#94019796 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7506862] (9) Sort [codegen id : 2] Input [2]: [date#94019796, drawdown_length_gross#94020028] Arguments: [date#94019796 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [2]: [date#94019796, drawdown_length_gross#94020028] Arguments: 1000000