== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [2]: [date#94262742, drawdown_length_gross#94262885] Arguments: [date#94262742, drawdown_length_gross#94262885] (2) InMemoryRelation Arguments: [date#94262742, drawdown_length_gross#94262885], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94262742 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94262742 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7526224] +- *(1) Project [date#94262742, (drawdown_length_gross#94262759 * -1) AS drawdown_length_gross#94262885] +- InMemoryTableScan [date#94262742, drawdown_length_gross#94262759] +- InMemoryRelation [date#94262742, allret#94262748, allcum#94262749, drawdown_gross#94262750, drawdown_length_gross#94262759, allretnet#94262760, allcumnet#94262761, drawdown_net#94262762, drawdown_length_net#94262768, numcos#94262769, largecum#94262770, largecumnet#94262800, largeret#94262802, largeretnet#94262808, midcum#94262809, midcumnet#94262815, midret#94262816, midretnet#94262822, smallcum#94262824, smallcumnet#94262825, smallret#94262831, smallretnet#94262832], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94262636 = null) THEN null ELSE cast(date#94262636 as date) END AS date#94262742, CASE WHEN ((allret#94262637 = NA) OR (allret#94262637 = null)) THEN null ELSE cast(allret#94262637 as float) END AS allret#94262748, CASE WHEN ((allcum#94262638 = NA) OR (allcum#94262638 = null)) THEN null ELSE cast(allcum#94262638 as float) END AS allcum#94262749, CASE WHEN ((drawdown_gross#94262639 = NA) OR (drawdown_gross#94262639 = null)) THEN null ELSE cast(drawdown_gross#94262639 as float) END AS drawdown_gross#94262750, CASE WHEN ((drawdown_length_gross#94262640 = NA) OR (drawdown_length_gross#94262640 = null)) THEN null ELSE cast(drawdown_length_gross#94262640 as int) END AS drawdown_length_gross#94262759, CASE WHEN ((allretnet#94262641 = NA) OR (allretnet#94262641 = null)) THEN null ELSE cast(allretnet#94262641 as float) END AS allretnet#94262760, CASE WHEN ((allcumnet#94262642 = NA) OR (allcumnet#94262642 = null)) THEN null ELSE cast(allcumnet#94262642 as float) END AS allcumnet#94262761, CASE WHEN ((drawdown_net#94262643 = NA) OR (drawdown_net#94262643 = null)) THEN null ELSE cast(drawdown_net#94262643 as float) END AS drawdown_net#94262762, CASE WHEN ((drawdown_length_net#94262644 = NA) OR (drawdown_length_net#94262644 = null)) THEN null ELSE cast(drawdown_length_net#94262644 as int) END AS drawdown_length_net#94262768, CASE WHEN ((numcos#94262645 = NA) OR (numcos#94262645 = null)) THEN null ELSE cast(numcos#94262645 as float) END AS numcos#94262769, CASE WHEN ((largecum#94262646 = NA) OR (largecum#94262646 = null)) THEN null ELSE cast(largecum#94262646 as float) END AS largecum#94262770, CASE WHEN ((largecumnet#94262647 = NA) OR (largecumnet#94262647 = null)) THEN null ELSE cast(largecumnet#94262647 as float) END AS largecumnet#94262800, CASE WHEN ((largeret#94262648 = NA) OR (largeret#94262648 = null)) THEN null ELSE cast(largeret#94262648 as float) END AS largeret#94262802, CASE WHEN ((largeretnet#94262649 = NA) OR (largeretnet#94262649 = null)) THEN null ELSE cast(largeretnet#94262649 as float) END AS largeretnet#94262808, CASE WHEN ((midcum#94262650 = NA) OR (midcum#94262650 = null)) THEN null ELSE cast(midcum#94262650 as float) END AS midcum#94262809, CASE WHEN ((midcumnet#94262651 = NA) OR (midcumnet#94262651 = null)) THEN null ELSE cast(midcumnet#94262651 as float) END AS midcumnet#94262815, CASE WHEN ((midret#94262652 = NA) OR (midret#94262652 = null)) THEN null ELSE cast(midret#94262652 as float) END AS midret#94262816, CASE WHEN ((midretnet#94262653 = NA) OR (midretnet#94262653 = null)) THEN null ELSE cast(midretnet#94262653 as float) END AS midretnet#94262822, CASE WHEN ((smallcum#94262654 = NA) OR (smallcum#94262654 = null)) THEN null ELSE cast(smallcum#94262654 as float) END AS smallcum#94262824, CASE WHEN ((smallcumnet#94262655 = NA) OR (smallcumnet#94262655 = null)) THEN null ELSE cast(smallcumnet#94262655 as float) END AS smallcumnet#94262825, CASE WHEN ((smallret#94262656 = NA) OR (smallret#94262656 = null)) THEN null ELSE cast(smallret#94262656 as float) END AS smallret#94262831, CASE WHEN ((smallretnet#94262657 = NA) OR (smallretnet#94262657 = null)) THEN null ELSE cast(smallretnet#94262657 as float) END AS smallretnet#94262832] +- FileScan csv [date#94262636,allret#94262637,allcum#94262638,drawdown_gross#94262639,drawdown_length_gross#94262640,allretnet#94262641,allcumnet#94262642,drawdown_net#94262643,drawdown_length_net#94262644,numcos#94262645,largecum#94262646,largecumnet#94262647,largeret#94262648,largeretnet#94262649,midcum#94262650,midcumnet#94262651,midret#94262652,midretnet#94262653,smallcum#94262654,smallcumnet#94262655,smallret#94262656,smallretnet#94262657] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/digital_revenue_signal..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94262742 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [date#94262742, drawdown_length_gross#94262759] Arguments: [date#94262742, drawdown_length_gross#94262759] (4) InMemoryRelation Arguments: [date#94262742, allret#94262748, allcum#94262749, drawdown_gross#94262750, drawdown_length_gross#94262759, allretnet#94262760, allcumnet#94262761, drawdown_net#94262762, drawdown_length_net#94262768, numcos#94262769, largecum#94262770, largecumnet#94262800, largeret#94262802, largeretnet#94262808, midcum#94262809, midcumnet#94262815, midret#94262816, midretnet#94262822, smallcum#94262824, smallcumnet#94262825, smallret#94262831, smallretnet#94262832], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94262636 = null) THEN null ELSE cast(date#94262636 as date) END AS date#94262742, CASE WHEN ((allret#94262637 = NA) OR (allret#94262637 = null)) THEN null ELSE cast(allret#94262637 as float) END AS allret#94262748, CASE WHEN ((allcum#94262638 = NA) OR (allcum#94262638 = null)) THEN null ELSE cast(allcum#94262638 as float) END AS allcum#94262749, CASE WHEN ((drawdown_gross#94262639 = NA) OR (drawdown_gross#94262639 = null)) THEN null ELSE cast(drawdown_gross#94262639 as float) END AS drawdown_gross#94262750, CASE WHEN ((drawdown_length_gross#94262640 = NA) OR (drawdown_length_gross#94262640 = null)) THEN null ELSE cast(drawdown_length_gross#94262640 as int) END AS drawdown_length_gross#94262759, CASE WHEN ((allretnet#94262641 = NA) OR (allretnet#94262641 = null)) THEN null ELSE cast(allretnet#94262641 as float) END AS allretnet#94262760, CASE WHEN ((allcumnet#94262642 = NA) OR (allcumnet#94262642 = null)) THEN null ELSE cast(allcumnet#94262642 as float) END AS allcumnet#94262761, CASE WHEN ((drawdown_net#94262643 = NA) OR (drawdown_net#94262643 = null)) THEN null ELSE cast(drawdown_net#94262643 as float) END AS drawdown_net#94262762, CASE WHEN ((drawdown_length_net#94262644 = NA) OR (drawdown_length_net#94262644 = null)) THEN null ELSE cast(drawdown_length_net#94262644 as int) END AS drawdown_length_net#94262768, CASE WHEN ((numcos#94262645 = NA) OR (numcos#94262645 = null)) THEN null ELSE cast(numcos#94262645 as float) END AS numcos#94262769, CASE WHEN ((largecum#94262646 = NA) OR (largecum#94262646 = null)) THEN null ELSE cast(largecum#94262646 as float) END AS largecum#94262770, CASE WHEN ((largecumnet#94262647 = NA) OR (largecumnet#94262647 = null)) THEN null ELSE cast(largecumnet#94262647 as float) END AS largecumnet#94262800, CASE WHEN ((largeret#94262648 = NA) OR (largeret#94262648 = null)) THEN null ELSE cast(largeret#94262648 as float) END AS largeret#94262802, CASE WHEN ((largeretnet#94262649 = NA) OR (largeretnet#94262649 = null)) THEN null ELSE cast(largeretnet#94262649 as float) END AS largeretnet#94262808, CASE WHEN ((midcum#94262650 = NA) OR (midcum#94262650 = null)) THEN null ELSE cast(midcum#94262650 as float) END AS midcum#94262809, CASE WHEN ((midcumnet#94262651 = NA) OR (midcumnet#94262651 = null)) THEN null ELSE cast(midcumnet#94262651 as float) END AS midcumnet#94262815, CASE WHEN ((midret#94262652 = NA) OR (midret#94262652 = null)) THEN null ELSE cast(midret#94262652 as float) END AS midret#94262816, CASE WHEN ((midretnet#94262653 = NA) OR (midretnet#94262653 = null)) THEN null ELSE cast(midretnet#94262653 as float) END AS midretnet#94262822, CASE WHEN ((smallcum#94262654 = NA) OR (smallcum#94262654 = null)) THEN null ELSE cast(smallcum#94262654 as float) END AS smallcum#94262824, CASE WHEN ((smallcumnet#94262655 = NA) OR (smallcumnet#94262655 = null)) THEN null ELSE cast(smallcumnet#94262655 as float) END AS smallcumnet#94262825, CASE WHEN ((smallret#94262656 = NA) OR (smallret#94262656 = null)) THEN null ELSE cast(smallret#94262656 as float) END AS smallret#94262831, CASE WHEN ((smallretnet#94262657 = NA) OR (smallretnet#94262657 = null)) THEN null ELSE cast(smallretnet#94262657 as float) END AS smallretnet#94262832] +- FileScan csv [date#94262636,allret#94262637,allcum#94262638,drawdown_gross#94262639,drawdown_length_gross#94262640,allretnet#94262641,allcumnet#94262642,drawdown_net#94262643,drawdown_length_net#94262644,numcos#94262645,largecum#94262646,largecumnet#94262647,largeret#94262648,largeretnet#94262649,midcum#94262650,midcumnet#94262651,midret#94262652,midretnet#94262653,smallcum#94262654,smallcumnet#94262655,smallret#94262656,smallretnet#94262657] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/digital_revenue_signal..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94262636, allret#94262637, allcum#94262638, drawdown_gross#94262639, drawdown_length_gross#94262640, allretnet#94262641, allcumnet#94262642, drawdown_net#94262643, drawdown_length_net#94262644, numcos#94262645, largecum#94262646, largecumnet#94262647, largeret#94262648, largeretnet#94262649, midcum#94262650, midcumnet#94262651, midret#94262652, midretnet#94262653, smallcum#94262654, smallcumnet#94262655, smallret#94262656, smallretnet#94262657] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/digital_revenue_signal/rev_signal_percentile_r_100/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94262636 = null) THEN null ELSE cast(date#94262636 as date) END AS date#94262742, CASE WHEN ((allret#94262637 = NA) OR (allret#94262637 = null)) THEN null ELSE cast(allret#94262637 as float) END AS allret#94262748, CASE WHEN ((allcum#94262638 = NA) OR (allcum#94262638 = null)) THEN null ELSE cast(allcum#94262638 as float) END AS allcum#94262749, CASE WHEN ((drawdown_gross#94262639 = NA) OR (drawdown_gross#94262639 = null)) THEN null ELSE cast(drawdown_gross#94262639 as float) END AS drawdown_gross#94262750, CASE WHEN ((drawdown_length_gross#94262640 = NA) OR (drawdown_length_gross#94262640 = null)) THEN null ELSE cast(drawdown_length_gross#94262640 as int) END AS drawdown_length_gross#94262759, CASE WHEN ((allretnet#94262641 = NA) OR (allretnet#94262641 = null)) THEN null ELSE cast(allretnet#94262641 as float) END AS allretnet#94262760, CASE WHEN ((allcumnet#94262642 = NA) OR (allcumnet#94262642 = null)) THEN null ELSE cast(allcumnet#94262642 as float) END AS allcumnet#94262761, CASE WHEN ((drawdown_net#94262643 = NA) OR (drawdown_net#94262643 = null)) THEN null ELSE cast(drawdown_net#94262643 as float) END AS drawdown_net#94262762, CASE WHEN ((drawdown_length_net#94262644 = NA) OR (drawdown_length_net#94262644 = null)) THEN null ELSE cast(drawdown_length_net#94262644 as int) END AS drawdown_length_net#94262768, CASE WHEN ((numcos#94262645 = NA) OR (numcos#94262645 = null)) THEN null ELSE cast(numcos#94262645 as float) END AS numcos#94262769, CASE WHEN ((largecum#94262646 = NA) OR (largecum#94262646 = null)) THEN null ELSE cast(largecum#94262646 as float) END AS largecum#94262770, CASE WHEN ((largecumnet#94262647 = NA) OR (largecumnet#94262647 = null)) THEN null ELSE cast(largecumnet#94262647 as float) END AS largecumnet#94262800, CASE WHEN ((largeret#94262648 = NA) OR (largeret#94262648 = null)) THEN null ELSE cast(largeret#94262648 as float) END AS largeret#94262802, CASE WHEN ((largeretnet#94262649 = NA) OR (largeretnet#94262649 = null)) THEN null ELSE cast(largeretnet#94262649 as float) END AS largeretnet#94262808, CASE WHEN ((midcum#94262650 = NA) OR (midcum#94262650 = null)) THEN null ELSE cast(midcum#94262650 as float) END AS midcum#94262809, CASE WHEN ((midcumnet#94262651 = NA) OR (midcumnet#94262651 = null)) THEN null ELSE cast(midcumnet#94262651 as float) END AS midcumnet#94262815, CASE WHEN ((midret#94262652 = NA) OR (midret#94262652 = null)) THEN null ELSE cast(midret#94262652 as float) END AS midret#94262816, CASE WHEN ((midretnet#94262653 = NA) OR (midretnet#94262653 = null)) THEN null ELSE cast(midretnet#94262653 as float) END AS midretnet#94262822, CASE WHEN ((smallcum#94262654 = NA) OR (smallcum#94262654 = null)) THEN null ELSE cast(smallcum#94262654 as float) END AS smallcum#94262824, CASE WHEN ((smallcumnet#94262655 = NA) OR (smallcumnet#94262655 = null)) THEN null ELSE cast(smallcumnet#94262655 as float) END AS smallcumnet#94262825, CASE WHEN ((smallret#94262656 = NA) OR (smallret#94262656 = null)) THEN null ELSE cast(smallret#94262656 as float) END AS smallret#94262831, CASE WHEN ((smallretnet#94262657 = NA) OR (smallretnet#94262657 = null)) THEN null ELSE cast(smallretnet#94262657 as float) END AS smallretnet#94262832] Input [22]: [date#94262636, allret#94262637, allcum#94262638, drawdown_gross#94262639, drawdown_length_gross#94262640, allretnet#94262641, allcumnet#94262642, drawdown_net#94262643, drawdown_length_net#94262644, numcos#94262645, largecum#94262646, largecumnet#94262647, largeret#94262648, largeretnet#94262649, midcum#94262650, midcumnet#94262651, midret#94262652, midretnet#94262653, smallcum#94262654, smallcumnet#94262655, smallret#94262656, smallretnet#94262657] (7) Project [codegen id : 1] Output [2]: [date#94262742, (drawdown_length_gross#94262759 * -1) AS drawdown_length_gross#94262885] Input [2]: [date#94262742, drawdown_length_gross#94262759] (8) Exchange Input [2]: [date#94262742, drawdown_length_gross#94262885] Arguments: rangepartitioning(date#94262742 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7526224] (9) Sort [codegen id : 2] Input [2]: [date#94262742, drawdown_length_gross#94262885] Arguments: [date#94262742 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [2]: [date#94262742, drawdown_length_gross#94262885] Arguments: 1000000