== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [2]: [date#94350024, drawdown_length_gross#94350068] Arguments: [date#94350024, drawdown_length_gross#94350068] (2) InMemoryRelation Arguments: [date#94350024, drawdown_length_gross#94350068], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94350024 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94350024 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7533114] +- *(1) Project [date#94350024, (drawdown_length_gross#94350028 * -1) AS drawdown_length_gross#94350068] +- InMemoryTableScan [date#94350024, drawdown_length_gross#94350028] +- InMemoryRelation [date#94350024, allret#94350025, allcum#94350026, drawdown_gross#94350027, drawdown_length_gross#94350028, allretnet#94350029, allcumnet#94350030, drawdown_net#94350031, drawdown_length_net#94350032, numcos#94350033, largecum#94350034, largecumnet#94350035, largeret#94350036, largeretnet#94350037, midcum#94350038, midcumnet#94350039, midret#94350040, midretnet#94350041, smallcum#94350042, smallcumnet#94350043, smallret#94350044, smallretnet#94350045], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94348290 = null) THEN null ELSE cast(date#94348290 as date) END AS date#94348458, CASE WHEN ((allret#94348291 = NA) OR (allret#94348291 = null)) THEN null ELSE cast(allret#94348291 as float) END AS allret#94348464, CASE WHEN ((allcum#94348292 = NA) OR (allcum#94348292 = null)) THEN null ELSE cast(allcum#94348292 as float) END AS allcum#94348470, CASE WHEN ((drawdown_gross#94348293 = NA) OR (drawdown_gross#94348293 = null)) THEN null ELSE cast(drawdown_gross#94348293 as float) END AS drawdown_gross#94348471, CASE WHEN ((drawdown_length_gross#94348294 = NA) OR (drawdown_length_gross#94348294 = null)) THEN null ELSE cast(drawdown_length_gross#94348294 as int) END AS drawdown_length_gross#94348477, CASE WHEN ((allretnet#94348295 = NA) OR (allretnet#94348295 = null)) THEN null ELSE cast(allretnet#94348295 as float) END AS allretnet#94348479, CASE WHEN ((allcumnet#94348296 = NA) OR (allcumnet#94348296 = null)) THEN null ELSE cast(allcumnet#94348296 as float) END AS allcumnet#94348502, CASE WHEN ((drawdown_net#94348297 = NA) OR (drawdown_net#94348297 = null)) THEN null ELSE cast(drawdown_net#94348297 as float) END AS drawdown_net#94348503, CASE WHEN ((drawdown_length_net#94348298 = NA) OR (drawdown_length_net#94348298 = null)) THEN null ELSE cast(drawdown_length_net#94348298 as int) END AS drawdown_length_net#94348509, CASE WHEN ((numcos#94348299 = NA) OR (numcos#94348299 = null)) THEN null ELSE cast(numcos#94348299 as float) END AS numcos#94348510, CASE WHEN ((largecum#94348300 = NA) OR (largecum#94348300 = null)) THEN null ELSE cast(largecum#94348300 as float) END AS largecum#94348511, CASE WHEN ((largecumnet#94348301 = NA) OR (largecumnet#94348301 = null)) THEN null ELSE cast(largecumnet#94348301 as float) END AS largecumnet#94348512, CASE WHEN ((largeret#94348302 = NA) OR (largeret#94348302 = null)) THEN null ELSE cast(largeret#94348302 as float) END AS largeret#94348539, CASE WHEN ((largeretnet#94348303 = NA) OR (largeretnet#94348303 = null)) THEN null ELSE cast(largeretnet#94348303 as float) END AS largeretnet#94348540, CASE WHEN ((midcum#94348304 = NA) OR (midcum#94348304 = null)) THEN null ELSE cast(midcum#94348304 as float) END AS midcum#94348554, CASE WHEN ((midcumnet#94348305 = NA) OR (midcumnet#94348305 = null)) THEN null ELSE cast(midcumnet#94348305 as float) END AS midcumnet#94348557, CASE WHEN ((midret#94348306 = NA) OR (midret#94348306 = null)) THEN null ELSE cast(midret#94348306 as float) END AS midret#94348563, CASE WHEN ((midretnet#94348307 = NA) OR (midretnet#94348307 = null)) THEN null ELSE cast(midretnet#94348307 as float) END AS midretnet#94348564, CASE WHEN ((smallcum#94348308 = NA) OR (smallcum#94348308 = null)) THEN null ELSE cast(smallcum#94348308 as float) END AS smallcum#94348565, CASE WHEN ((smallcumnet#94348309 = NA) OR (smallcumnet#94348309 = null)) THEN null ELSE cast(smallcumnet#94348309 as float) END AS smallcumnet#94348566, CASE WHEN ((smallret#94348310 = NA) OR (smallret#94348310 = null)) THEN null ELSE cast(smallret#94348310 as float) END AS smallret#94348567, CASE WHEN ((smallretnet#94348311 = NA) OR (smallretnet#94348311 = null)) THEN null ELSE cast(smallretnet#94348311 as float) END AS smallretnet#94348569] +- FileScan csv [date#94348290,allret#94348291,allcum#94348292,drawdown_gross#94348293,drawdown_length_gross#94348294,allretnet#94348295,allcumnet#94348296,drawdown_net#94348297,drawdown_length_net#94348298,numcos#94348299,largecum#94348300,largecumnet#94348301,largeret#94348302,largeretnet#94348303,midcum#94348304,midcumnet#94348305,midret#94348306,midretnet#94348307,smallcum#94348308,smallcumnet#94348309,smallret#94348310,smallretnet#94348311] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94350024 ASC NULLS FIRST] (3) InMemoryTableScan Output [2]: [date#94350024, drawdown_length_gross#94350028] Arguments: [date#94350024, drawdown_length_gross#94350028] (4) InMemoryRelation Arguments: [date#94350024, allret#94350025, allcum#94350026, drawdown_gross#94350027, drawdown_length_gross#94350028, allretnet#94350029, allcumnet#94350030, drawdown_net#94350031, drawdown_length_net#94350032, numcos#94350033, largecum#94350034, largecumnet#94350035, largeret#94350036, largeretnet#94350037, midcum#94350038, midcumnet#94350039, midret#94350040, midretnet#94350041, smallcum#94350042, smallcumnet#94350043, smallret#94350044, smallretnet#94350045], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94348290 = null) THEN null ELSE cast(date#94348290 as date) END AS date#94348458, CASE WHEN ((allret#94348291 = NA) OR (allret#94348291 = null)) THEN null ELSE cast(allret#94348291 as float) END AS allret#94348464, CASE WHEN ((allcum#94348292 = NA) OR (allcum#94348292 = null)) THEN null ELSE cast(allcum#94348292 as float) END AS allcum#94348470, CASE WHEN ((drawdown_gross#94348293 = NA) OR (drawdown_gross#94348293 = null)) THEN null ELSE cast(drawdown_gross#94348293 as float) END AS drawdown_gross#94348471, CASE WHEN ((drawdown_length_gross#94348294 = NA) OR (drawdown_length_gross#94348294 = null)) THEN null ELSE cast(drawdown_length_gross#94348294 as int) END AS drawdown_length_gross#94348477, CASE WHEN ((allretnet#94348295 = NA) OR (allretnet#94348295 = null)) THEN null ELSE cast(allretnet#94348295 as float) END AS allretnet#94348479, CASE WHEN ((allcumnet#94348296 = NA) OR (allcumnet#94348296 = null)) THEN null ELSE cast(allcumnet#94348296 as float) END AS allcumnet#94348502, CASE WHEN ((drawdown_net#94348297 = NA) OR (drawdown_net#94348297 = null)) THEN null ELSE cast(drawdown_net#94348297 as float) END AS drawdown_net#94348503, CASE WHEN ((drawdown_length_net#94348298 = NA) OR (drawdown_length_net#94348298 = null)) THEN null ELSE cast(drawdown_length_net#94348298 as int) END AS drawdown_length_net#94348509, CASE WHEN ((numcos#94348299 = NA) OR (numcos#94348299 = null)) THEN null ELSE cast(numcos#94348299 as float) END AS numcos#94348510, CASE WHEN ((largecum#94348300 = NA) OR (largecum#94348300 = null)) THEN null ELSE cast(largecum#94348300 as float) END AS largecum#94348511, CASE WHEN ((largecumnet#94348301 = NA) OR (largecumnet#94348301 = null)) THEN null ELSE cast(largecumnet#94348301 as float) END AS largecumnet#94348512, CASE WHEN ((largeret#94348302 = NA) OR (largeret#94348302 = null)) THEN null ELSE cast(largeret#94348302 as float) END AS largeret#94348539, CASE WHEN ((largeretnet#94348303 = NA) OR (largeretnet#94348303 = null)) THEN null ELSE cast(largeretnet#94348303 as float) END AS largeretnet#94348540, CASE WHEN ((midcum#94348304 = NA) OR (midcum#94348304 = null)) THEN null ELSE cast(midcum#94348304 as float) END AS midcum#94348554, CASE WHEN ((midcumnet#94348305 = NA) OR (midcumnet#94348305 = null)) THEN null ELSE cast(midcumnet#94348305 as float) END AS midcumnet#94348557, CASE WHEN ((midret#94348306 = NA) OR (midret#94348306 = null)) THEN null ELSE cast(midret#94348306 as float) END AS midret#94348563, CASE WHEN ((midretnet#94348307 = NA) OR (midretnet#94348307 = null)) THEN null ELSE cast(midretnet#94348307 as float) END AS midretnet#94348564, CASE WHEN ((smallcum#94348308 = NA) OR (smallcum#94348308 = null)) THEN null ELSE cast(smallcum#94348308 as float) END AS smallcum#94348565, CASE WHEN ((smallcumnet#94348309 = NA) OR (smallcumnet#94348309 = null)) THEN null ELSE cast(smallcumnet#94348309 as float) END AS smallcumnet#94348566, CASE WHEN ((smallret#94348310 = NA) OR (smallret#94348310 = null)) THEN null ELSE cast(smallret#94348310 as float) END AS smallret#94348567, CASE WHEN ((smallretnet#94348311 = NA) OR (smallretnet#94348311 = null)) THEN null ELSE cast(smallretnet#94348311 as float) END AS smallretnet#94348569] +- FileScan csv [date#94348290,allret#94348291,allcum#94348292,drawdown_gross#94348293,drawdown_length_gross#94348294,allretnet#94348295,allcumnet#94348296,drawdown_net#94348297,drawdown_length_net#94348298,numcos#94348299,largecum#94348300,largecumnet#94348301,largeret#94348302,largeretnet#94348303,midcum#94348304,midcumnet#94348305,midret#94348306,midretnet#94348307,smallcum#94348308,smallcumnet#94348309,smallret#94348310,smallretnet#94348311] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94348290, allret#94348291, allcum#94348292, drawdown_gross#94348293, drawdown_length_gross#94348294, allretnet#94348295, allcumnet#94348296, drawdown_net#94348297, drawdown_length_net#94348298, numcos#94348299, largecum#94348300, largecumnet#94348301, largeret#94348302, largeretnet#94348303, midcum#94348304, midcumnet#94348305, midret#94348306, midretnet#94348307, smallcum#94348308, smallcumnet#94348309, smallret#94348310, smallretnet#94348311] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rankviews/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94348290 = null) THEN null ELSE cast(date#94348290 as date) END AS date#94348458, CASE WHEN ((allret#94348291 = NA) OR (allret#94348291 = null)) THEN null ELSE cast(allret#94348291 as float) END AS allret#94348464, CASE WHEN ((allcum#94348292 = NA) OR (allcum#94348292 = null)) THEN null ELSE cast(allcum#94348292 as float) END AS allcum#94348470, CASE WHEN ((drawdown_gross#94348293 = NA) OR (drawdown_gross#94348293 = null)) THEN null ELSE cast(drawdown_gross#94348293 as float) END AS drawdown_gross#94348471, CASE WHEN ((drawdown_length_gross#94348294 = NA) OR (drawdown_length_gross#94348294 = null)) THEN null ELSE cast(drawdown_length_gross#94348294 as int) END AS drawdown_length_gross#94348477, CASE WHEN ((allretnet#94348295 = NA) OR (allretnet#94348295 = null)) THEN null ELSE cast(allretnet#94348295 as float) END AS allretnet#94348479, CASE WHEN ((allcumnet#94348296 = NA) OR (allcumnet#94348296 = null)) THEN null ELSE cast(allcumnet#94348296 as float) END AS allcumnet#94348502, CASE WHEN ((drawdown_net#94348297 = NA) OR (drawdown_net#94348297 = null)) THEN null ELSE cast(drawdown_net#94348297 as float) END AS drawdown_net#94348503, CASE WHEN ((drawdown_length_net#94348298 = NA) OR (drawdown_length_net#94348298 = null)) THEN null ELSE cast(drawdown_length_net#94348298 as int) END AS drawdown_length_net#94348509, CASE WHEN ((numcos#94348299 = NA) OR (numcos#94348299 = null)) THEN null ELSE cast(numcos#94348299 as float) END AS numcos#94348510, CASE WHEN ((largecum#94348300 = NA) OR (largecum#94348300 = null)) THEN null ELSE cast(largecum#94348300 as float) END AS largecum#94348511, CASE WHEN ((largecumnet#94348301 = NA) OR (largecumnet#94348301 = null)) THEN null ELSE cast(largecumnet#94348301 as float) END AS largecumnet#94348512, CASE WHEN ((largeret#94348302 = NA) OR (largeret#94348302 = null)) THEN null ELSE cast(largeret#94348302 as float) END AS largeret#94348539, CASE WHEN ((largeretnet#94348303 = NA) OR (largeretnet#94348303 = null)) THEN null ELSE cast(largeretnet#94348303 as float) END AS largeretnet#94348540, CASE WHEN ((midcum#94348304 = NA) OR (midcum#94348304 = null)) THEN null ELSE cast(midcum#94348304 as float) END AS midcum#94348554, CASE WHEN ((midcumnet#94348305 = NA) OR (midcumnet#94348305 = null)) THEN null ELSE cast(midcumnet#94348305 as float) END AS midcumnet#94348557, CASE WHEN ((midret#94348306 = NA) OR (midret#94348306 = null)) THEN null ELSE cast(midret#94348306 as float) END AS midret#94348563, CASE WHEN ((midretnet#94348307 = NA) OR (midretnet#94348307 = null)) THEN null ELSE cast(midretnet#94348307 as float) END AS midretnet#94348564, CASE WHEN ((smallcum#94348308 = NA) OR (smallcum#94348308 = null)) THEN null ELSE cast(smallcum#94348308 as float) END AS smallcum#94348565, CASE WHEN ((smallcumnet#94348309 = NA) OR (smallcumnet#94348309 = null)) THEN null ELSE cast(smallcumnet#94348309 as float) END AS smallcumnet#94348566, CASE WHEN ((smallret#94348310 = NA) OR (smallret#94348310 = null)) THEN null ELSE cast(smallret#94348310 as float) END AS smallret#94348567, CASE WHEN ((smallretnet#94348311 = NA) OR (smallretnet#94348311 = null)) THEN null ELSE cast(smallretnet#94348311 as float) END AS smallretnet#94348569] Input [22]: [date#94348290, allret#94348291, allcum#94348292, drawdown_gross#94348293, drawdown_length_gross#94348294, allretnet#94348295, allcumnet#94348296, drawdown_net#94348297, drawdown_length_net#94348298, numcos#94348299, largecum#94348300, largecumnet#94348301, largeret#94348302, largeretnet#94348303, midcum#94348304, midcumnet#94348305, midret#94348306, midretnet#94348307, smallcum#94348308, smallcumnet#94348309, smallret#94348310, smallretnet#94348311] (7) Project [codegen id : 1] Output [2]: [date#94350024, (drawdown_length_gross#94350028 * -1) AS drawdown_length_gross#94350068] Input [2]: [date#94350024, drawdown_length_gross#94350028] (8) Exchange Input [2]: [date#94350024, drawdown_length_gross#94350068] Arguments: rangepartitioning(date#94350024 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7533114] (9) Sort [codegen id : 2] Input [2]: [date#94350024, drawdown_length_gross#94350068] Arguments: [date#94350024 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [2]: [date#94350024, drawdown_length_gross#94350068] Arguments: 1000000