== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [date#94006072, cum_gross#94134471, cum_net#94134472] Arguments: [date#94006072, cum_gross#94134471, cum_net#94134472] (2) InMemoryRelation Arguments: [date#94006072, cum_gross#94134471, cum_net#94134472], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94006073 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94006073 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7505704] +- *(1) Project [date#94006073, allcum#94006077 AS cum_gross#94006163, allcumnet#94006086 AS cum_net#94006164] +- InMemoryTableScan [allcum#94006077, allcumnet#94006086, date#94006073] +- InMemoryRelation [date#94006073, allret#94006075, allcum#94006077, drawdown_gross#94006079, drawdown_length_gross#94006082, allretnet#94006084, allcumnet#94006086, drawdown_net#94006088, drawdown_length_net#94006090, numcos#94006092, largecum#94006094, largecumnet#94006097, largeret#94006099, largeretnet#94006102, midcum#94006103, midcumnet#94006105, midret#94006106, midretnet#94006108, smallcum#94006110, smallcumnet#94006112, smallret#94006113, smallretnet#94006115], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94005174 = null) THEN null ELSE cast(date#94005174 as date) END AS date#94005335, CASE WHEN ((allret#94005175 = NA) OR (allret#94005175 = null)) THEN null ELSE cast(allret#94005175 as float) END AS allret#94005336, CASE WHEN ((allcum#94005176 = NA) OR (allcum#94005176 = null)) THEN null ELSE cast(allcum#94005176 as float) END AS allcum#94005360, CASE WHEN ((drawdown_gross#94005177 = NA) OR (drawdown_gross#94005177 = null)) THEN null ELSE cast(drawdown_gross#94005177 as float) END AS drawdown_gross#94005361, CASE WHEN ((drawdown_length_gross#94005178 = NA) OR (drawdown_length_gross#94005178 = null)) THEN null ELSE cast(drawdown_length_gross#94005178 as int) END AS drawdown_length_gross#94005385, CASE WHEN ((allretnet#94005179 = NA) OR (allretnet#94005179 = null)) THEN null ELSE cast(allretnet#94005179 as float) END AS allretnet#94005389, CASE WHEN ((allcumnet#94005180 = NA) OR (allcumnet#94005180 = null)) THEN null ELSE cast(allcumnet#94005180 as float) END AS allcumnet#94005392, CASE WHEN ((drawdown_net#94005181 = NA) OR (drawdown_net#94005181 = null)) THEN null ELSE cast(drawdown_net#94005181 as float) END AS drawdown_net#94005396, CASE WHEN ((drawdown_length_net#94005182 = NA) OR (drawdown_length_net#94005182 = null)) THEN null ELSE cast(drawdown_length_net#94005182 as int) END AS drawdown_length_net#94005405, CASE WHEN ((numcos#94005183 = NA) OR (numcos#94005183 = null)) THEN null ELSE cast(numcos#94005183 as float) END AS numcos#94005411, CASE WHEN ((largecum#94005184 = NA) OR (largecum#94005184 = null)) THEN null ELSE cast(largecum#94005184 as float) END AS largecum#94005417, CASE WHEN ((largecumnet#94005185 = NA) OR (largecumnet#94005185 = null)) THEN null ELSE cast(largecumnet#94005185 as float) END AS largecumnet#94005426, CASE WHEN ((largeret#94005186 = NA) OR (largeret#94005186 = null)) THEN null ELSE cast(largeret#94005186 as float) END AS largeret#94005428, CASE WHEN ((largeretnet#94005187 = NA) OR (largeretnet#94005187 = null)) THEN null ELSE cast(largeretnet#94005187 as float) END AS largeretnet#94005430, CASE WHEN ((midcum#94005188 = NA) OR (midcum#94005188 = null)) THEN null ELSE cast(midcum#94005188 as float) END AS midcum#94005432, CASE WHEN ((midcumnet#94005189 = NA) OR (midcumnet#94005189 = null)) THEN null ELSE cast(midcumnet#94005189 as float) END AS midcumnet#94005434, CASE WHEN ((midret#94005190 = NA) OR (midret#94005190 = null)) THEN null ELSE cast(midret#94005190 as float) END AS midret#94005438, CASE WHEN ((midretnet#94005191 = NA) OR (midretnet#94005191 = null)) THEN null ELSE cast(midretnet#94005191 as float) END AS midretnet#94005441, CASE WHEN ((smallcum#94005192 = NA) OR (smallcum#94005192 = null)) THEN null ELSE cast(smallcum#94005192 as float) END AS smallcum#94005442, CASE WHEN ((smallcumnet#94005193 = NA) OR (smallcumnet#94005193 = null)) THEN null ELSE cast(smallcumnet#94005193 as float) END AS smallcumnet#94005444, CASE WHEN ((smallret#94005194 = NA) OR (smallret#94005194 = null)) THEN null ELSE cast(smallret#94005194 as float) END AS smallret#94005447, CASE WHEN ((smallretnet#94005195 = NA) OR (smallretnet#94005195 = null)) THEN null ELSE cast(smallretnet#94005195 as float) END AS smallretnet#94005448] +- FileScan csv [date#94005174,allret#94005175,allcum#94005176,drawdown_gross#94005177,drawdown_length_gross#94005178,allretnet#94005179,allcumnet#94005180,drawdown_net#94005181,drawdown_length_net#94005182,numcos#94005183,largecum#94005184,largecumnet#94005185,largeret#94005186,largeretnet#94005187,midcum#94005188,midcumnet#94005189,midret#94005190,midretnet#94005191,smallcum#94005192,smallcumnet#94005193,smallret#94005194,smallretnet#94005195] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94006073 ASC NULLS FIRST] (3) InMemoryTableScan Output [3]: [allcum#94006077, allcumnet#94006086, date#94006073] Arguments: [allcum#94006077, allcumnet#94006086, date#94006073] (4) InMemoryRelation Arguments: [date#94006073, allret#94006075, allcum#94006077, drawdown_gross#94006079, drawdown_length_gross#94006082, allretnet#94006084, allcumnet#94006086, drawdown_net#94006088, drawdown_length_net#94006090, numcos#94006092, largecum#94006094, largecumnet#94006097, largeret#94006099, largeretnet#94006102, midcum#94006103, midcumnet#94006105, midret#94006106, midretnet#94006108, smallcum#94006110, smallcumnet#94006112, smallret#94006113, smallretnet#94006115], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94005174 = null) THEN null ELSE cast(date#94005174 as date) END AS date#94005335, CASE WHEN ((allret#94005175 = NA) OR (allret#94005175 = null)) THEN null ELSE cast(allret#94005175 as float) END AS allret#94005336, CASE WHEN ((allcum#94005176 = NA) OR (allcum#94005176 = null)) THEN null ELSE cast(allcum#94005176 as float) END AS allcum#94005360, CASE WHEN ((drawdown_gross#94005177 = NA) OR (drawdown_gross#94005177 = null)) THEN null ELSE cast(drawdown_gross#94005177 as float) END AS drawdown_gross#94005361, CASE WHEN ((drawdown_length_gross#94005178 = NA) OR (drawdown_length_gross#94005178 = null)) THEN null ELSE cast(drawdown_length_gross#94005178 as int) END AS drawdown_length_gross#94005385, CASE WHEN ((allretnet#94005179 = NA) OR (allretnet#94005179 = null)) THEN null ELSE cast(allretnet#94005179 as float) END AS allretnet#94005389, CASE WHEN ((allcumnet#94005180 = NA) OR (allcumnet#94005180 = null)) THEN null ELSE cast(allcumnet#94005180 as float) END AS allcumnet#94005392, CASE WHEN ((drawdown_net#94005181 = NA) OR (drawdown_net#94005181 = null)) THEN null ELSE cast(drawdown_net#94005181 as float) END AS drawdown_net#94005396, CASE WHEN ((drawdown_length_net#94005182 = NA) OR (drawdown_length_net#94005182 = null)) THEN null ELSE cast(drawdown_length_net#94005182 as int) END AS drawdown_length_net#94005405, CASE WHEN ((numcos#94005183 = NA) OR (numcos#94005183 = null)) THEN null ELSE cast(numcos#94005183 as float) END AS numcos#94005411, CASE WHEN ((largecum#94005184 = NA) OR (largecum#94005184 = null)) THEN null ELSE cast(largecum#94005184 as float) END AS largecum#94005417, CASE WHEN ((largecumnet#94005185 = NA) OR (largecumnet#94005185 = null)) THEN null ELSE cast(largecumnet#94005185 as float) END AS largecumnet#94005426, CASE WHEN ((largeret#94005186 = NA) OR (largeret#94005186 = null)) THEN null ELSE cast(largeret#94005186 as float) END AS largeret#94005428, CASE WHEN ((largeretnet#94005187 = NA) OR (largeretnet#94005187 = null)) THEN null ELSE cast(largeretnet#94005187 as float) END AS largeretnet#94005430, CASE WHEN ((midcum#94005188 = NA) OR (midcum#94005188 = null)) THEN null ELSE cast(midcum#94005188 as float) END AS midcum#94005432, CASE WHEN ((midcumnet#94005189 = NA) OR (midcumnet#94005189 = null)) THEN null ELSE cast(midcumnet#94005189 as float) END AS midcumnet#94005434, CASE WHEN ((midret#94005190 = NA) OR (midret#94005190 = null)) THEN null ELSE cast(midret#94005190 as float) END AS midret#94005438, CASE WHEN ((midretnet#94005191 = NA) OR (midretnet#94005191 = null)) THEN null ELSE cast(midretnet#94005191 as float) END AS midretnet#94005441, CASE WHEN ((smallcum#94005192 = NA) OR (smallcum#94005192 = null)) THEN null ELSE cast(smallcum#94005192 as float) END AS smallcum#94005442, CASE WHEN ((smallcumnet#94005193 = NA) OR (smallcumnet#94005193 = null)) THEN null ELSE cast(smallcumnet#94005193 as float) END AS smallcumnet#94005444, CASE WHEN ((smallret#94005194 = NA) OR (smallret#94005194 = null)) THEN null ELSE cast(smallret#94005194 as float) END AS smallret#94005447, CASE WHEN ((smallretnet#94005195 = NA) OR (smallretnet#94005195 = null)) THEN null ELSE cast(smallretnet#94005195 as float) END AS smallretnet#94005448] +- FileScan csv [date#94005174,allret#94005175,allcum#94005176,drawdown_gross#94005177,drawdown_length_gross#94005178,allretnet#94005179,allcumnet#94005180,drawdown_net#94005181,drawdown_length_net#94005182,numcos#94005183,largecum#94005184,largecumnet#94005185,largeret#94005186,largeretnet#94005187,midcum#94005188,midcumnet#94005189,midret#94005190,midretnet#94005191,smallcum#94005192,smallcumnet#94005193,smallret#94005194,smallretnet#94005195] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94005174, allret#94005175, allcum#94005176, drawdown_gross#94005177, drawdown_length_gross#94005178, allretnet#94005179, allcumnet#94005180, drawdown_net#94005181, drawdown_length_net#94005182, numcos#94005183, largecum#94005184, largecumnet#94005185, largeret#94005186, largeretnet#94005187, midcum#94005188, midcumnet#94005189, midret#94005190, midretnet#94005191, smallcum#94005192, smallcumnet#94005193, smallret#94005194, smallretnet#94005195] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rankviews/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94005174 = null) THEN null ELSE cast(date#94005174 as date) END AS date#94005335, CASE WHEN ((allret#94005175 = NA) OR (allret#94005175 = null)) THEN null ELSE cast(allret#94005175 as float) END AS allret#94005336, CASE WHEN ((allcum#94005176 = NA) OR (allcum#94005176 = null)) THEN null ELSE cast(allcum#94005176 as float) END AS allcum#94005360, CASE WHEN ((drawdown_gross#94005177 = NA) OR (drawdown_gross#94005177 = null)) THEN null ELSE cast(drawdown_gross#94005177 as float) END AS drawdown_gross#94005361, CASE WHEN ((drawdown_length_gross#94005178 = NA) OR (drawdown_length_gross#94005178 = null)) THEN null ELSE cast(drawdown_length_gross#94005178 as int) END AS drawdown_length_gross#94005385, CASE WHEN ((allretnet#94005179 = NA) OR (allretnet#94005179 = null)) THEN null ELSE cast(allretnet#94005179 as float) END AS allretnet#94005389, CASE WHEN ((allcumnet#94005180 = NA) OR (allcumnet#94005180 = null)) THEN null ELSE cast(allcumnet#94005180 as float) END AS allcumnet#94005392, CASE WHEN ((drawdown_net#94005181 = NA) OR (drawdown_net#94005181 = null)) THEN null ELSE cast(drawdown_net#94005181 as float) END AS drawdown_net#94005396, CASE WHEN ((drawdown_length_net#94005182 = NA) OR (drawdown_length_net#94005182 = null)) THEN null ELSE cast(drawdown_length_net#94005182 as int) END AS drawdown_length_net#94005405, CASE WHEN ((numcos#94005183 = NA) OR (numcos#94005183 = null)) THEN null ELSE cast(numcos#94005183 as float) END AS numcos#94005411, CASE WHEN ((largecum#94005184 = NA) OR (largecum#94005184 = null)) THEN null ELSE cast(largecum#94005184 as float) END AS largecum#94005417, CASE WHEN ((largecumnet#94005185 = NA) OR (largecumnet#94005185 = null)) THEN null ELSE cast(largecumnet#94005185 as float) END AS largecumnet#94005426, CASE WHEN ((largeret#94005186 = NA) OR (largeret#94005186 = null)) THEN null ELSE cast(largeret#94005186 as float) END AS largeret#94005428, CASE WHEN ((largeretnet#94005187 = NA) OR (largeretnet#94005187 = null)) THEN null ELSE cast(largeretnet#94005187 as float) END AS largeretnet#94005430, CASE WHEN ((midcum#94005188 = NA) OR (midcum#94005188 = null)) THEN null ELSE cast(midcum#94005188 as float) END AS midcum#94005432, CASE WHEN ((midcumnet#94005189 = NA) OR (midcumnet#94005189 = null)) THEN null ELSE cast(midcumnet#94005189 as float) END AS midcumnet#94005434, CASE WHEN ((midret#94005190 = NA) OR (midret#94005190 = null)) THEN null ELSE cast(midret#94005190 as float) END AS midret#94005438, CASE WHEN ((midretnet#94005191 = NA) OR (midretnet#94005191 = null)) THEN null ELSE cast(midretnet#94005191 as float) END AS midretnet#94005441, CASE WHEN ((smallcum#94005192 = NA) OR (smallcum#94005192 = null)) THEN null ELSE cast(smallcum#94005192 as float) END AS smallcum#94005442, CASE WHEN ((smallcumnet#94005193 = NA) OR (smallcumnet#94005193 = null)) THEN null ELSE cast(smallcumnet#94005193 as float) END AS smallcumnet#94005444, CASE WHEN ((smallret#94005194 = NA) OR (smallret#94005194 = null)) THEN null ELSE cast(smallret#94005194 as float) END AS smallret#94005447, CASE WHEN ((smallretnet#94005195 = NA) OR (smallretnet#94005195 = null)) THEN null ELSE cast(smallretnet#94005195 as float) END AS smallretnet#94005448] Input [22]: [date#94005174, allret#94005175, allcum#94005176, drawdown_gross#94005177, drawdown_length_gross#94005178, allretnet#94005179, allcumnet#94005180, drawdown_net#94005181, drawdown_length_net#94005182, numcos#94005183, largecum#94005184, largecumnet#94005185, largeret#94005186, largeretnet#94005187, midcum#94005188, midcumnet#94005189, midret#94005190, midretnet#94005191, smallcum#94005192, smallcumnet#94005193, smallret#94005194, smallretnet#94005195] (7) Project [codegen id : 1] Output [3]: [date#94006073, allcum#94006077 AS cum_gross#94006163, allcumnet#94006086 AS cum_net#94006164] Input [3]: [allcum#94006077, allcumnet#94006086, date#94006073] (8) Exchange Input [3]: [date#94006073, cum_gross#94006163, cum_net#94006164] Arguments: rangepartitioning(date#94006073 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7505704] (9) Sort [codegen id : 2] Input [3]: [date#94006073, cum_gross#94006163, cum_net#94006164] Arguments: [date#94006073 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [3]: [date#94006072, cum_gross#94134471, cum_net#94134472] Arguments: 1000000