== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [date#93991976, daily_gross#93992043, daily_net#93992044] Arguments: [date#93991976, daily_gross#93992043, daily_net#93992044] (2) InMemoryRelation Arguments: [date#93991976, daily_gross#93992043, daily_net#93992044], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#93991976 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#93991976 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7504579] +- *(1) Project [date#93991976, allret#93991977 AS daily_gross#93992043, allretnet#93991981 AS daily_net#93992044] +- InMemoryTableScan [allret#93991977, allretnet#93991981, date#93991976] +- InMemoryRelation [date#93991976, allret#93991977, allcum#93991978, drawdown_gross#93991979, drawdown_length_gross#93991980, allretnet#93991981, allcumnet#93991982, drawdown_net#93991983, drawdown_length_net#93991984, numcos#93991985, largecum#93991986, largecumnet#93991987, largeret#93991988, largeretnet#93991989, midcum#93991990, midcumnet#93991991, midret#93991992, midretnet#93991993, smallcum#93991994, smallcumnet#93991995, smallret#93991996, smallretnet#93991997], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#93990419 = null) THEN null ELSE cast(date#93990419 as date) END AS date#93990517, CASE WHEN ((allret#93990420 = NA) OR (allret#93990420 = null)) THEN null ELSE cast(allret#93990420 as float) END AS allret#93990519, CASE WHEN ((allcum#93990421 = NA) OR (allcum#93990421 = null)) THEN null ELSE cast(allcum#93990421 as float) END AS allcum#93990521, CASE WHEN ((drawdown_gross#93990422 = NA) OR (drawdown_gross#93990422 = null)) THEN null ELSE cast(drawdown_gross#93990422 as float) END AS drawdown_gross#93990523, CASE WHEN ((drawdown_length_gross#93990423 = NA) OR (drawdown_length_gross#93990423 = null)) THEN null ELSE cast(drawdown_length_gross#93990423 as int) END AS drawdown_length_gross#93990526, CASE WHEN ((allretnet#93990424 = NA) OR (allretnet#93990424 = null)) THEN null ELSE cast(allretnet#93990424 as float) END AS allretnet#93990528, CASE WHEN ((allcumnet#93990425 = NA) OR (allcumnet#93990425 = null)) THEN null ELSE cast(allcumnet#93990425 as float) END AS allcumnet#93990530, CASE WHEN ((drawdown_net#93990426 = NA) OR (drawdown_net#93990426 = null)) THEN null ELSE cast(drawdown_net#93990426 as float) END AS drawdown_net#93990533, CASE WHEN ((drawdown_length_net#93990427 = NA) OR (drawdown_length_net#93990427 = null)) THEN null ELSE cast(drawdown_length_net#93990427 as int) END AS drawdown_length_net#93990534, CASE WHEN ((numcos#93990428 = NA) OR (numcos#93990428 = null)) THEN null ELSE cast(numcos#93990428 as float) END AS numcos#93990536, CASE WHEN ((largecum#93990429 = NA) OR (largecum#93990429 = null)) THEN null ELSE cast(largecum#93990429 as float) END AS largecum#93990539, CASE WHEN ((largecumnet#93990430 = NA) OR (largecumnet#93990430 = null)) THEN null ELSE cast(largecumnet#93990430 as float) END AS largecumnet#93990541, CASE WHEN ((largeret#93990431 = NA) OR (largeret#93990431 = null)) THEN null ELSE cast(largeret#93990431 as float) END AS largeret#93990544, CASE WHEN ((largeretnet#93990432 = NA) OR (largeretnet#93990432 = null)) THEN null ELSE cast(largeretnet#93990432 as float) END AS largeretnet#93990545, CASE WHEN ((midcum#93990433 = NA) OR (midcum#93990433 = null)) THEN null ELSE cast(midcum#93990433 as float) END AS midcum#93990546, CASE WHEN ((midcumnet#93990434 = NA) OR (midcumnet#93990434 = null)) THEN null ELSE cast(midcumnet#93990434 as float) END AS midcumnet#93990547, CASE WHEN ((midret#93990435 = NA) OR (midret#93990435 = null)) THEN null ELSE cast(midret#93990435 as float) END AS midret#93990548, CASE WHEN ((midretnet#93990436 = NA) OR (midretnet#93990436 = null)) THEN null ELSE cast(midretnet#93990436 as float) END AS midretnet#93990549, CASE WHEN ((smallcum#93990437 = NA) OR (smallcum#93990437 = null)) THEN null ELSE cast(smallcum#93990437 as float) END AS smallcum#93990550, CASE WHEN ((smallcumnet#93990438 = NA) OR (smallcumnet#93990438 = null)) THEN null ELSE cast(smallcumnet#93990438 as float) END AS smallcumnet#93990551, CASE WHEN ((smallret#93990439 = NA) OR (smallret#93990439 = null)) THEN null ELSE cast(smallret#93990439 as float) END AS smallret#93990552, CASE WHEN ((smallretnet#93990440 = NA) OR (smallretnet#93990440 = null)) THEN null ELSE cast(smallretnet#93990440 as float) END AS smallretnet#93990553] +- FileScan csv [date#93990419,allret#93990420,allcum#93990421,drawdown_gross#93990422,drawdown_length_gross#93990423,allretnet#93990424,allcumnet#93990425,drawdown_net#93990426,drawdown_length_net#93990427,numcos#93990428,largecum#93990429,largecumnet#93990430,largeret#93990431,largeretnet#93990432,midcum#93990433,midcumnet#93990434,midret#93990435,midretnet#93990436,smallcum#93990437,smallcumnet#93990438,smallret#93990439,smallretnet#93990440] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#93991976 ASC NULLS FIRST] (3) InMemoryTableScan Output [3]: [allret#93991977, allretnet#93991981, date#93991976] Arguments: [allret#93991977, allretnet#93991981, date#93991976] (4) InMemoryRelation Arguments: [date#93991976, allret#93991977, allcum#93991978, drawdown_gross#93991979, drawdown_length_gross#93991980, allretnet#93991981, allcumnet#93991982, drawdown_net#93991983, drawdown_length_net#93991984, numcos#93991985, largecum#93991986, largecumnet#93991987, largeret#93991988, largeretnet#93991989, midcum#93991990, midcumnet#93991991, midret#93991992, midretnet#93991993, smallcum#93991994, smallcumnet#93991995, smallret#93991996, smallretnet#93991997], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#93990419 = null) THEN null ELSE cast(date#93990419 as date) END AS date#93990517, CASE WHEN ((allret#93990420 = NA) OR (allret#93990420 = null)) THEN null ELSE cast(allret#93990420 as float) END AS allret#93990519, CASE WHEN ((allcum#93990421 = NA) OR (allcum#93990421 = null)) THEN null ELSE cast(allcum#93990421 as float) END AS allcum#93990521, CASE WHEN ((drawdown_gross#93990422 = NA) OR (drawdown_gross#93990422 = null)) THEN null ELSE cast(drawdown_gross#93990422 as float) END AS drawdown_gross#93990523, CASE WHEN ((drawdown_length_gross#93990423 = NA) OR (drawdown_length_gross#93990423 = null)) THEN null ELSE cast(drawdown_length_gross#93990423 as int) END AS drawdown_length_gross#93990526, CASE WHEN ((allretnet#93990424 = NA) OR (allretnet#93990424 = null)) THEN null ELSE cast(allretnet#93990424 as float) END AS allretnet#93990528, CASE WHEN ((allcumnet#93990425 = NA) OR (allcumnet#93990425 = null)) THEN null ELSE cast(allcumnet#93990425 as float) END AS allcumnet#93990530, CASE WHEN ((drawdown_net#93990426 = NA) OR (drawdown_net#93990426 = null)) THEN null ELSE cast(drawdown_net#93990426 as float) END AS drawdown_net#93990533, CASE WHEN ((drawdown_length_net#93990427 = NA) OR (drawdown_length_net#93990427 = null)) THEN null ELSE cast(drawdown_length_net#93990427 as int) END AS drawdown_length_net#93990534, CASE WHEN ((numcos#93990428 = NA) OR (numcos#93990428 = null)) THEN null ELSE cast(numcos#93990428 as float) END AS numcos#93990536, CASE WHEN ((largecum#93990429 = NA) OR (largecum#93990429 = null)) THEN null ELSE cast(largecum#93990429 as float) END AS largecum#93990539, CASE WHEN ((largecumnet#93990430 = NA) OR (largecumnet#93990430 = null)) THEN null ELSE cast(largecumnet#93990430 as float) END AS largecumnet#93990541, CASE WHEN ((largeret#93990431 = NA) OR (largeret#93990431 = null)) THEN null ELSE cast(largeret#93990431 as float) END AS largeret#93990544, CASE WHEN ((largeretnet#93990432 = NA) OR (largeretnet#93990432 = null)) THEN null ELSE cast(largeretnet#93990432 as float) END AS largeretnet#93990545, CASE WHEN ((midcum#93990433 = NA) OR (midcum#93990433 = null)) THEN null ELSE cast(midcum#93990433 as float) END AS midcum#93990546, CASE WHEN ((midcumnet#93990434 = NA) OR (midcumnet#93990434 = null)) THEN null ELSE cast(midcumnet#93990434 as float) END AS midcumnet#93990547, CASE WHEN ((midret#93990435 = NA) OR (midret#93990435 = null)) THEN null ELSE cast(midret#93990435 as float) END AS midret#93990548, CASE WHEN ((midretnet#93990436 = NA) OR (midretnet#93990436 = null)) THEN null ELSE cast(midretnet#93990436 as float) END AS midretnet#93990549, CASE WHEN ((smallcum#93990437 = NA) OR (smallcum#93990437 = null)) THEN null ELSE cast(smallcum#93990437 as float) END AS smallcum#93990550, CASE WHEN ((smallcumnet#93990438 = NA) OR (smallcumnet#93990438 = null)) THEN null ELSE cast(smallcumnet#93990438 as float) END AS smallcumnet#93990551, CASE WHEN ((smallret#93990439 = NA) OR (smallret#93990439 = null)) THEN null ELSE cast(smallret#93990439 as float) END AS smallret#93990552, CASE WHEN ((smallretnet#93990440 = NA) OR (smallretnet#93990440 = null)) THEN null ELSE cast(smallretnet#93990440 as float) END AS smallretnet#93990553] +- FileScan csv [date#93990419,allret#93990420,allcum#93990421,drawdown_gross#93990422,drawdown_length_gross#93990423,allretnet#93990424,allcumnet#93990425,drawdown_net#93990426,drawdown_length_net#93990427,numcos#93990428,largecum#93990429,largecumnet#93990430,largeret#93990431,largeretnet#93990432,midcum#93990433,midcumnet#93990434,midret#93990435,midretnet#93990436,smallcum#93990437,smallcumnet#93990438,smallret#93990439,smallretnet#93990440] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#93990419, allret#93990420, allcum#93990421, drawdown_gross#93990422, drawdown_length_gross#93990423, allretnet#93990424, allcumnet#93990425, drawdown_net#93990426, drawdown_length_net#93990427, numcos#93990428, largecum#93990429, largecumnet#93990430, largeret#93990431, largeretnet#93990432, midcum#93990433, midcumnet#93990434, midret#93990435, midretnet#93990436, smallcum#93990437, smallcumnet#93990438, smallret#93990439, smallretnet#93990440] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/estimize_signal_history/estimizesignal_preearnings/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#93990419 = null) THEN null ELSE cast(date#93990419 as date) END AS date#93990517, CASE WHEN ((allret#93990420 = NA) OR (allret#93990420 = null)) THEN null ELSE cast(allret#93990420 as float) END AS allret#93990519, CASE WHEN ((allcum#93990421 = NA) OR (allcum#93990421 = null)) THEN null ELSE cast(allcum#93990421 as float) END AS allcum#93990521, CASE WHEN ((drawdown_gross#93990422 = NA) OR (drawdown_gross#93990422 = null)) THEN null ELSE cast(drawdown_gross#93990422 as float) END AS drawdown_gross#93990523, CASE WHEN ((drawdown_length_gross#93990423 = NA) OR (drawdown_length_gross#93990423 = null)) THEN null ELSE cast(drawdown_length_gross#93990423 as int) END AS drawdown_length_gross#93990526, CASE WHEN ((allretnet#93990424 = NA) OR (allretnet#93990424 = null)) THEN null ELSE cast(allretnet#93990424 as float) END AS allretnet#93990528, CASE WHEN ((allcumnet#93990425 = NA) OR (allcumnet#93990425 = null)) THEN null ELSE cast(allcumnet#93990425 as float) END AS allcumnet#93990530, CASE WHEN ((drawdown_net#93990426 = NA) OR (drawdown_net#93990426 = null)) THEN null ELSE cast(drawdown_net#93990426 as float) END AS drawdown_net#93990533, CASE WHEN ((drawdown_length_net#93990427 = NA) OR (drawdown_length_net#93990427 = null)) THEN null ELSE cast(drawdown_length_net#93990427 as int) END AS drawdown_length_net#93990534, CASE WHEN ((numcos#93990428 = NA) OR (numcos#93990428 = null)) THEN null ELSE cast(numcos#93990428 as float) END AS numcos#93990536, CASE WHEN ((largecum#93990429 = NA) OR (largecum#93990429 = null)) THEN null ELSE cast(largecum#93990429 as float) END AS largecum#93990539, CASE WHEN ((largecumnet#93990430 = NA) OR (largecumnet#93990430 = null)) THEN null ELSE cast(largecumnet#93990430 as float) END AS largecumnet#93990541, CASE WHEN ((largeret#93990431 = NA) OR (largeret#93990431 = null)) THEN null ELSE cast(largeret#93990431 as float) END AS largeret#93990544, CASE WHEN ((largeretnet#93990432 = NA) OR (largeretnet#93990432 = null)) THEN null ELSE cast(largeretnet#93990432 as float) END AS largeretnet#93990545, CASE WHEN ((midcum#93990433 = NA) OR (midcum#93990433 = null)) THEN null ELSE cast(midcum#93990433 as float) END AS midcum#93990546, CASE WHEN ((midcumnet#93990434 = NA) OR (midcumnet#93990434 = null)) THEN null ELSE cast(midcumnet#93990434 as float) END AS midcumnet#93990547, CASE WHEN ((midret#93990435 = NA) OR (midret#93990435 = null)) THEN null ELSE cast(midret#93990435 as float) END AS midret#93990548, CASE WHEN ((midretnet#93990436 = NA) OR (midretnet#93990436 = null)) THEN null ELSE cast(midretnet#93990436 as float) END AS midretnet#93990549, CASE WHEN ((smallcum#93990437 = NA) OR (smallcum#93990437 = null)) THEN null ELSE cast(smallcum#93990437 as float) END AS smallcum#93990550, CASE WHEN ((smallcumnet#93990438 = NA) OR (smallcumnet#93990438 = null)) THEN null ELSE cast(smallcumnet#93990438 as float) END AS smallcumnet#93990551, CASE WHEN ((smallret#93990439 = NA) OR (smallret#93990439 = null)) THEN null ELSE cast(smallret#93990439 as float) END AS smallret#93990552, CASE WHEN ((smallretnet#93990440 = NA) OR (smallretnet#93990440 = null)) THEN null ELSE cast(smallretnet#93990440 as float) END AS smallretnet#93990553] Input [22]: [date#93990419, allret#93990420, allcum#93990421, drawdown_gross#93990422, drawdown_length_gross#93990423, allretnet#93990424, allcumnet#93990425, drawdown_net#93990426, drawdown_length_net#93990427, numcos#93990428, largecum#93990429, largecumnet#93990430, largeret#93990431, largeretnet#93990432, midcum#93990433, midcumnet#93990434, midret#93990435, midretnet#93990436, smallcum#93990437, smallcumnet#93990438, smallret#93990439, smallretnet#93990440] (7) Project [codegen id : 1] Output [3]: [date#93991976, allret#93991977 AS daily_gross#93992043, allretnet#93991981 AS daily_net#93992044] Input [3]: [allret#93991977, allretnet#93991981, date#93991976] (8) Exchange Input [3]: [date#93991976, daily_gross#93992043, daily_net#93992044] Arguments: rangepartitioning(date#93991976 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7504579] (9) Sort [codegen id : 2] Input [3]: [date#93991976, daily_gross#93992043, daily_net#93992044] Arguments: [date#93991976 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [3]: [date#93991976, daily_gross#93992043, daily_net#93992044] Arguments: 1000000