== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [2]: [date#93990517, drawdown_length_gross#93990626] Arguments: [date#93990517, drawdown_length_gross#93990626] (2) InMemoryRelation Arguments: [date#93990517, drawdown_length_gross#93990626], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#93990517 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#93990517 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7504619] +- *(1) Project [date#93990517, (drawdown_length_gross#93990526 * -1) AS drawdown_length_gross#93990626] +- InMemoryTableScan [date#93990517, drawdown_length_gross#93990526] +- InMemoryRelation [date#93990517, allret#93990519, allcum#93990521, drawdown_gross#93990523, drawdown_length_gross#93990526, allretnet#93990528, allcumnet#93990530, drawdown_net#93990533, drawdown_length_net#93990534, numcos#93990536, largecum#93990539, largecumnet#93990541, largeret#93990544, largeretnet#93990545, midcum#93990546, midcumnet#93990547, midret#93990548, midretnet#93990549, smallcum#93990550, smallcumnet#93990551, smallret#93990552, smallretnet#93990553], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#93990419 = null) THEN null ELSE cast(date#93990419 as date) END AS date#93990517, CASE WHEN ((allret#93990420 = NA) OR (allret#93990420 = null)) THEN null ELSE cast(allret#93990420 as float) END AS allret#93990519, CASE WHEN ((allcum#93990421 = NA) OR (allcum#93990421 = null)) THEN null ELSE cast(allcum#93990421 as float) END AS allcum#93990521, CASE WHEN ((drawdown_gross#93990422 = NA) OR (drawdown_gross#93990422 = null)) THEN null ELSE cast(drawdown_gross#93990422 as float) END AS drawdown_gross#93990523, CASE WHEN ((drawdown_length_gross#93990423 = NA) OR (drawdown_length_gross#93990423 = null)) THEN null ELSE cast(drawdown_length_gross#93990423 as int) END AS drawdown_length_gross#93990526, CASE WHEN ((allretnet#93990424 = NA) OR (allretnet#93990424 = null)) THEN null ELSE cast(allretnet#93990424 as float) END AS allretnet#93990528, CASE WHEN ((allcumnet#93990425 = NA) OR (allcumnet#93990425 = null)) THEN null ELSE cast(allcumnet#93990425 as float) END AS allcumnet#93990530, CASE WHEN ((drawdown_net#93990426 = NA) OR (drawdown_net#93990426 = null)) THEN null ELSE cast(drawdown_net#93990426 as float) END AS drawdown_net#93990533, CASE WHEN ((drawdown_length_net#93990427 = NA) OR (drawdown_length_net#93990427 = null)) THEN null ELSE cast(drawdown_length_net#93990427 as int) END AS drawdown_length_net#93990534, CASE WHEN ((numcos#93990428 = NA) OR (numcos#93990428 = null)) THEN null ELSE cast(numcos#93990428 as float) END AS numcos#93990536, CASE WHEN ((largecum#93990429 = NA) OR (largecum#93990429 = null)) THEN null ELSE cast(largecum#93990429 as float) END AS largecum#93990539, CASE WHEN ((largecumnet#93990430 = NA) OR (largecumnet#93990430 = null)) THEN null ELSE cast(largecumnet#93990430 as float) END AS largecumnet#93990541, CASE WHEN ((largeret#93990431 = NA) OR (largeret#93990431 = null)) THEN null ELSE cast(largeret#93990431 as float) END AS largeret#93990544, CASE WHEN ((largeretnet#93990432 = NA) OR (largeretnet#93990432 = null)) THEN null ELSE cast(largeretnet#93990432 as float) END AS largeretnet#93990545, CASE WHEN ((midcum#93990433 = NA) OR (midcum#93990433 = null)) THEN null ELSE cast(midcum#93990433 as float) END AS midcum#93990546, CASE WHEN ((midcumnet#93990434 = NA) OR (midcumnet#93990434 = null)) THEN null ELSE cast(midcumnet#93990434 as float) END AS midcumnet#93990547, CASE WHEN ((midret#93990435 = NA) OR (midret#93990435 = null)) THEN null ELSE cast(midret#93990435 as float) END AS midret#93990548, CASE WHEN ((midretnet#93990436 = NA) OR (midretnet#93990436 = null)) THEN null ELSE cast(midretnet#93990436 as float) END AS midretnet#93990549, CASE WHEN ((smallcum#93990437 = NA) OR (smallcum#93990437 = null)) THEN null ELSE cast(smallcum#93990437 as float) END AS smallcum#93990550, CASE WHEN ((smallcumnet#93990438 = NA) OR (smallcumnet#93990438 = null)) THEN null ELSE cast(smallcumnet#93990438 as float) END AS smallcumnet#93990551, CASE WHEN ((smallret#93990439 = NA) OR (smallret#93990439 = null)) THEN null ELSE cast(smallret#93990439 as float) END AS smallret#93990552, CASE WHEN ((smallretnet#93990440 = NA) OR (smallretnet#93990440 = null)) THEN null ELSE cast(smallretnet#93990440 as float) END AS smallretnet#93990553] +- FileScan csv [date#93990419,allret#93990420,allcum#93990421,drawdown_gross#93990422,drawdown_length_gross#93990423,allretnet#93990424,allcumnet#93990425,drawdown_net#93990426,drawdown_length_net#93990427,numcos#93990428,largecum#93990429,largecumnet#93990430,largeret#93990431,largeretnet#93990432,midcum#93990433,midcumnet#93990434,midret#93990435,midretnet#93990436,smallcum#93990437,smallcumnet#93990438,smallret#93990439,smallretnet#93990440] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#93990517 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [date#93990517, drawdown_length_gross#93990526] Arguments: [date#93990517, drawdown_length_gross#93990526] (4) InMemoryRelation Arguments: [date#93990517, allret#93990519, allcum#93990521, drawdown_gross#93990523, drawdown_length_gross#93990526, allretnet#93990528, allcumnet#93990530, drawdown_net#93990533, drawdown_length_net#93990534, numcos#93990536, largecum#93990539, largecumnet#93990541, largeret#93990544, largeretnet#93990545, midcum#93990546, midcumnet#93990547, midret#93990548, midretnet#93990549, smallcum#93990550, smallcumnet#93990551, smallret#93990552, smallretnet#93990553], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#93990419 = null) THEN null ELSE cast(date#93990419 as date) END AS date#93990517, CASE WHEN ((allret#93990420 = NA) OR (allret#93990420 = null)) THEN null ELSE cast(allret#93990420 as float) END AS allret#93990519, CASE WHEN ((allcum#93990421 = NA) OR (allcum#93990421 = null)) THEN null ELSE cast(allcum#93990421 as float) END AS allcum#93990521, CASE WHEN ((drawdown_gross#93990422 = NA) OR (drawdown_gross#93990422 = null)) THEN null ELSE cast(drawdown_gross#93990422 as float) END AS drawdown_gross#93990523, CASE WHEN ((drawdown_length_gross#93990423 = NA) OR (drawdown_length_gross#93990423 = null)) THEN null ELSE cast(drawdown_length_gross#93990423 as int) END AS drawdown_length_gross#93990526, CASE WHEN ((allretnet#93990424 = NA) OR (allretnet#93990424 = null)) THEN null ELSE cast(allretnet#93990424 as float) END AS allretnet#93990528, CASE WHEN ((allcumnet#93990425 = NA) OR (allcumnet#93990425 = null)) THEN null ELSE cast(allcumnet#93990425 as float) END AS allcumnet#93990530, CASE WHEN ((drawdown_net#93990426 = NA) OR (drawdown_net#93990426 = null)) THEN null ELSE cast(drawdown_net#93990426 as float) END AS drawdown_net#93990533, CASE WHEN ((drawdown_length_net#93990427 = NA) OR (drawdown_length_net#93990427 = null)) THEN null ELSE cast(drawdown_length_net#93990427 as int) END AS drawdown_length_net#93990534, CASE WHEN ((numcos#93990428 = NA) OR (numcos#93990428 = null)) THEN null ELSE cast(numcos#93990428 as float) END AS numcos#93990536, CASE WHEN ((largecum#93990429 = NA) OR (largecum#93990429 = null)) THEN null ELSE cast(largecum#93990429 as float) END AS largecum#93990539, CASE WHEN ((largecumnet#93990430 = NA) OR (largecumnet#93990430 = null)) THEN null ELSE cast(largecumnet#93990430 as float) END AS largecumnet#93990541, CASE WHEN ((largeret#93990431 = NA) OR (largeret#93990431 = null)) THEN null ELSE cast(largeret#93990431 as float) END AS largeret#93990544, CASE WHEN ((largeretnet#93990432 = NA) OR (largeretnet#93990432 = null)) THEN null ELSE cast(largeretnet#93990432 as float) END AS largeretnet#93990545, CASE WHEN ((midcum#93990433 = NA) OR (midcum#93990433 = null)) THEN null ELSE cast(midcum#93990433 as float) END AS midcum#93990546, CASE WHEN ((midcumnet#93990434 = NA) OR (midcumnet#93990434 = null)) THEN null ELSE cast(midcumnet#93990434 as float) END AS midcumnet#93990547, CASE WHEN ((midret#93990435 = NA) OR (midret#93990435 = null)) THEN null ELSE cast(midret#93990435 as float) END AS midret#93990548, CASE WHEN ((midretnet#93990436 = NA) OR (midretnet#93990436 = null)) THEN null ELSE cast(midretnet#93990436 as float) END AS midretnet#93990549, CASE WHEN ((smallcum#93990437 = NA) OR (smallcum#93990437 = null)) THEN null ELSE cast(smallcum#93990437 as float) END AS smallcum#93990550, CASE WHEN ((smallcumnet#93990438 = NA) OR (smallcumnet#93990438 = null)) THEN null ELSE cast(smallcumnet#93990438 as float) END AS smallcumnet#93990551, CASE WHEN ((smallret#93990439 = NA) OR (smallret#93990439 = null)) THEN null ELSE cast(smallret#93990439 as float) END AS smallret#93990552, CASE WHEN ((smallretnet#93990440 = NA) OR (smallretnet#93990440 = null)) THEN null ELSE cast(smallretnet#93990440 as float) END AS smallretnet#93990553] +- FileScan csv [date#93990419,allret#93990420,allcum#93990421,drawdown_gross#93990422,drawdown_length_gross#93990423,allretnet#93990424,allcumnet#93990425,drawdown_net#93990426,drawdown_length_net#93990427,numcos#93990428,largecum#93990429,largecumnet#93990430,largeret#93990431,largeretnet#93990432,midcum#93990433,midcumnet#93990434,midret#93990435,midretnet#93990436,smallcum#93990437,smallcumnet#93990438,smallret#93990439,smallretnet#93990440] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#93990419, allret#93990420, allcum#93990421, drawdown_gross#93990422, drawdown_length_gross#93990423, allretnet#93990424, allcumnet#93990425, drawdown_net#93990426, drawdown_length_net#93990427, numcos#93990428, largecum#93990429, largecumnet#93990430, largeret#93990431, largeretnet#93990432, midcum#93990433, midcumnet#93990434, midret#93990435, midretnet#93990436, smallcum#93990437, smallcumnet#93990438, smallret#93990439, smallretnet#93990440] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/estimize_signal_history/estimizesignal_preearnings/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#93990419 = null) THEN null ELSE cast(date#93990419 as date) END AS date#93990517, CASE WHEN ((allret#93990420 = NA) OR (allret#93990420 = null)) THEN null ELSE cast(allret#93990420 as float) END AS allret#93990519, CASE WHEN ((allcum#93990421 = NA) OR (allcum#93990421 = null)) THEN null ELSE cast(allcum#93990421 as float) END AS allcum#93990521, CASE WHEN ((drawdown_gross#93990422 = NA) OR (drawdown_gross#93990422 = null)) THEN null ELSE cast(drawdown_gross#93990422 as float) END AS drawdown_gross#93990523, CASE WHEN ((drawdown_length_gross#93990423 = NA) OR (drawdown_length_gross#93990423 = null)) THEN null ELSE cast(drawdown_length_gross#93990423 as int) END AS drawdown_length_gross#93990526, CASE WHEN ((allretnet#93990424 = NA) OR (allretnet#93990424 = null)) THEN null ELSE cast(allretnet#93990424 as float) END AS allretnet#93990528, CASE WHEN ((allcumnet#93990425 = NA) OR (allcumnet#93990425 = null)) THEN null ELSE cast(allcumnet#93990425 as float) END AS allcumnet#93990530, CASE WHEN ((drawdown_net#93990426 = NA) OR (drawdown_net#93990426 = null)) THEN null ELSE cast(drawdown_net#93990426 as float) END AS drawdown_net#93990533, CASE WHEN ((drawdown_length_net#93990427 = NA) OR (drawdown_length_net#93990427 = null)) THEN null ELSE cast(drawdown_length_net#93990427 as int) END AS drawdown_length_net#93990534, CASE WHEN ((numcos#93990428 = NA) OR (numcos#93990428 = null)) THEN null ELSE cast(numcos#93990428 as float) END AS numcos#93990536, CASE WHEN ((largecum#93990429 = NA) OR (largecum#93990429 = null)) THEN null ELSE cast(largecum#93990429 as float) END AS largecum#93990539, CASE WHEN ((largecumnet#93990430 = NA) OR (largecumnet#93990430 = null)) THEN null ELSE cast(largecumnet#93990430 as float) END AS largecumnet#93990541, CASE WHEN ((largeret#93990431 = NA) OR (largeret#93990431 = null)) THEN null ELSE cast(largeret#93990431 as float) END AS largeret#93990544, CASE WHEN ((largeretnet#93990432 = NA) OR (largeretnet#93990432 = null)) THEN null ELSE cast(largeretnet#93990432 as float) END AS largeretnet#93990545, CASE WHEN ((midcum#93990433 = NA) OR (midcum#93990433 = null)) THEN null ELSE cast(midcum#93990433 as float) END AS midcum#93990546, CASE WHEN ((midcumnet#93990434 = NA) OR (midcumnet#93990434 = null)) THEN null ELSE cast(midcumnet#93990434 as float) END AS midcumnet#93990547, CASE WHEN ((midret#93990435 = NA) OR (midret#93990435 = null)) THEN null ELSE cast(midret#93990435 as float) END AS midret#93990548, CASE WHEN ((midretnet#93990436 = NA) OR (midretnet#93990436 = null)) THEN null ELSE cast(midretnet#93990436 as float) END AS midretnet#93990549, CASE WHEN ((smallcum#93990437 = NA) OR (smallcum#93990437 = null)) THEN null ELSE cast(smallcum#93990437 as float) END AS smallcum#93990550, CASE WHEN ((smallcumnet#93990438 = NA) OR (smallcumnet#93990438 = null)) THEN null ELSE cast(smallcumnet#93990438 as float) END AS smallcumnet#93990551, CASE WHEN ((smallret#93990439 = NA) OR (smallret#93990439 = null)) THEN null ELSE cast(smallret#93990439 as float) END AS smallret#93990552, CASE WHEN ((smallretnet#93990440 = NA) OR (smallretnet#93990440 = null)) THEN null ELSE cast(smallretnet#93990440 as float) END AS smallretnet#93990553] Input [22]: [date#93990419, allret#93990420, allcum#93990421, drawdown_gross#93990422, drawdown_length_gross#93990423, allretnet#93990424, allcumnet#93990425, drawdown_net#93990426, drawdown_length_net#93990427, numcos#93990428, largecum#93990429, largecumnet#93990430, largeret#93990431, largeretnet#93990432, midcum#93990433, midcumnet#93990434, midret#93990435, midretnet#93990436, smallcum#93990437, smallcumnet#93990438, smallret#93990439, smallretnet#93990440] (7) Project [codegen id : 1] Output [2]: [date#93990517, (drawdown_length_gross#93990526 * -1) AS drawdown_length_gross#93990626] Input [2]: [date#93990517, drawdown_length_gross#93990526] (8) Exchange Input [2]: [date#93990517, drawdown_length_gross#93990626] Arguments: rangepartitioning(date#93990517 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7504619] (9) Sort [codegen id : 2] Input [2]: [date#93990517, drawdown_length_gross#93990626] Arguments: [date#93990517 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [2]: [date#93990517, drawdown_length_gross#93990626] Arguments: 1000000