== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [date#94334879, daily_gross#94335166, daily_net#94335167] Arguments: [date#94334879, daily_gross#94335166, daily_net#94335167] (2) InMemoryRelation Arguments: [date#94334879, daily_gross#94335166, daily_net#94335167], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94334879 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94334879 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7531921] +- *(1) Project [date#94334879, allret#94334882 AS daily_gross#94335166, allretnet#94334892 AS daily_net#94335167] +- InMemoryTableScan [allret#94334882, allretnet#94334892, date#94334879] +- InMemoryRelation [date#94334879, allret#94334882, allcum#94334884, drawdown_gross#94334886, drawdown_length_gross#94334889, allretnet#94334892, allcumnet#94334895, drawdown_net#94334898, drawdown_length_net#94334900, numcos#94334902, largecum#94334904, largecumnet#94334906, largeret#94334931, largeretnet#94334932, midcum#94334933, midcumnet#94334934, midret#94334935, midretnet#94334936, smallcum#94334962, smallcumnet#94334963, smallret#94334975, smallretnet#94334977], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94333860 = null) THEN null ELSE cast(date#94333860 as date) END AS date#94333951, CASE WHEN ((allret#94333861 = NA) OR (allret#94333861 = null)) THEN null ELSE cast(allret#94333861 as float) END AS allret#94333952, CASE WHEN ((allcum#94333862 = NA) OR (allcum#94333862 = null)) THEN null ELSE cast(allcum#94333862 as float) END AS allcum#94333953, CASE WHEN ((drawdown_gross#94333863 = NA) OR (drawdown_gross#94333863 = null)) THEN null ELSE cast(drawdown_gross#94333863 as float) END AS drawdown_gross#94333954, CASE WHEN ((drawdown_length_gross#94333864 = NA) OR (drawdown_length_gross#94333864 = null)) THEN null ELSE cast(drawdown_length_gross#94333864 as int) END AS drawdown_length_gross#94333955, CASE WHEN ((allretnet#94333865 = NA) OR (allretnet#94333865 = null)) THEN null ELSE cast(allretnet#94333865 as float) END AS allretnet#94333956, CASE WHEN ((allcumnet#94333866 = NA) OR (allcumnet#94333866 = null)) THEN null ELSE cast(allcumnet#94333866 as float) END AS allcumnet#94333957, CASE WHEN ((drawdown_net#94333867 = NA) OR (drawdown_net#94333867 = null)) THEN null ELSE cast(drawdown_net#94333867 as float) END AS drawdown_net#94333958, CASE WHEN ((drawdown_length_net#94333868 = NA) OR (drawdown_length_net#94333868 = null)) THEN null ELSE cast(drawdown_length_net#94333868 as int) END AS drawdown_length_net#94333964, CASE WHEN ((numcos#94333869 = NA) OR (numcos#94333869 = null)) THEN null ELSE cast(numcos#94333869 as float) END AS numcos#94333970, CASE WHEN ((largecum#94333870 = NA) OR (largecum#94333870 = null)) THEN null ELSE cast(largecum#94333870 as float) END AS largecum#94333976, CASE WHEN ((largecumnet#94333871 = NA) OR (largecumnet#94333871 = null)) THEN null ELSE cast(largecumnet#94333871 as float) END AS largecumnet#94333983, CASE WHEN ((largeret#94333872 = NA) OR (largeret#94333872 = null)) THEN null ELSE cast(largeret#94333872 as float) END AS largeret#94333984, CASE WHEN ((largeretnet#94333873 = NA) OR (largeretnet#94333873 = null)) THEN null ELSE cast(largeretnet#94333873 as float) END AS largeretnet#94333985, CASE WHEN ((midcum#94333874 = NA) OR (midcum#94333874 = null)) THEN null ELSE cast(midcum#94333874 as float) END AS midcum#94333986, CASE WHEN ((midcumnet#94333875 = NA) OR (midcumnet#94333875 = null)) THEN null ELSE cast(midcumnet#94333875 as float) END AS midcumnet#94333987, CASE WHEN ((midret#94333876 = NA) OR (midret#94333876 = null)) THEN null ELSE cast(midret#94333876 as float) END AS midret#94333988, CASE WHEN ((midretnet#94333877 = NA) OR (midretnet#94333877 = null)) THEN null ELSE cast(midretnet#94333877 as float) END AS midretnet#94334011, CASE WHEN ((smallcum#94333878 = NA) OR (smallcum#94333878 = null)) THEN null ELSE cast(smallcum#94333878 as float) END AS smallcum#94334017, CASE WHEN ((smallcumnet#94333879 = NA) OR (smallcumnet#94333879 = null)) THEN null ELSE cast(smallcumnet#94333879 as float) END AS smallcumnet#94334045, CASE WHEN ((smallret#94333880 = NA) OR (smallret#94333880 = null)) THEN null ELSE cast(smallret#94333880 as float) END AS smallret#94334051, CASE WHEN ((smallretnet#94333881 = NA) OR (smallretnet#94333881 = null)) THEN null ELSE cast(smallretnet#94333881 as float) END AS smallretnet#94334110] +- FileScan csv [date#94333860,allret#94333861,allcum#94333862,drawdown_gross#94333863,drawdown_length_gross#94333864,allretnet#94333865,allcumnet#94333866,drawdown_net#94333867,drawdown_length_net#94333868,numcos#94333869,largecum#94333870,largecumnet#94333871,largeret#94333872,largeretnet#94333873,midcum#94333874,midcumnet#94333875,midret#94333876,midretnet#94333877,smallcum#94333878,smallcumnet#94333879,smallret#94333880,smallretnet#94333881] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94334879 ASC NULLS FIRST] (3) InMemoryTableScan Output [3]: [allret#94334882, allretnet#94334892, date#94334879] Arguments: [allret#94334882, allretnet#94334892, date#94334879] (4) InMemoryRelation Arguments: [date#94334879, allret#94334882, allcum#94334884, drawdown_gross#94334886, drawdown_length_gross#94334889, allretnet#94334892, allcumnet#94334895, drawdown_net#94334898, drawdown_length_net#94334900, numcos#94334902, largecum#94334904, largecumnet#94334906, largeret#94334931, largeretnet#94334932, midcum#94334933, midcumnet#94334934, midret#94334935, midretnet#94334936, smallcum#94334962, smallcumnet#94334963, smallret#94334975, smallretnet#94334977], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94333860 = null) THEN null ELSE cast(date#94333860 as date) END AS date#94333951, CASE WHEN ((allret#94333861 = NA) OR (allret#94333861 = null)) THEN null ELSE cast(allret#94333861 as float) END AS allret#94333952, CASE WHEN ((allcum#94333862 = NA) OR (allcum#94333862 = null)) THEN null ELSE cast(allcum#94333862 as float) END AS allcum#94333953, CASE WHEN ((drawdown_gross#94333863 = NA) OR (drawdown_gross#94333863 = null)) THEN null ELSE cast(drawdown_gross#94333863 as float) END AS drawdown_gross#94333954, CASE WHEN ((drawdown_length_gross#94333864 = NA) OR (drawdown_length_gross#94333864 = null)) THEN null ELSE cast(drawdown_length_gross#94333864 as int) END AS drawdown_length_gross#94333955, CASE WHEN ((allretnet#94333865 = NA) OR (allretnet#94333865 = null)) THEN null ELSE cast(allretnet#94333865 as float) END AS allretnet#94333956, CASE WHEN ((allcumnet#94333866 = NA) OR (allcumnet#94333866 = null)) THEN null ELSE cast(allcumnet#94333866 as float) END AS allcumnet#94333957, CASE WHEN ((drawdown_net#94333867 = NA) OR (drawdown_net#94333867 = null)) THEN null ELSE cast(drawdown_net#94333867 as float) END AS drawdown_net#94333958, CASE WHEN ((drawdown_length_net#94333868 = NA) OR (drawdown_length_net#94333868 = null)) THEN null ELSE cast(drawdown_length_net#94333868 as int) END AS drawdown_length_net#94333964, CASE WHEN ((numcos#94333869 = NA) OR (numcos#94333869 = null)) THEN null ELSE cast(numcos#94333869 as float) END AS numcos#94333970, CASE WHEN ((largecum#94333870 = NA) OR (largecum#94333870 = null)) THEN null ELSE cast(largecum#94333870 as float) END AS largecum#94333976, CASE WHEN ((largecumnet#94333871 = NA) OR (largecumnet#94333871 = null)) THEN null ELSE cast(largecumnet#94333871 as float) END AS largecumnet#94333983, CASE WHEN ((largeret#94333872 = NA) OR (largeret#94333872 = null)) THEN null ELSE cast(largeret#94333872 as float) END AS largeret#94333984, CASE WHEN ((largeretnet#94333873 = NA) OR (largeretnet#94333873 = null)) THEN null ELSE cast(largeretnet#94333873 as float) END AS largeretnet#94333985, CASE WHEN ((midcum#94333874 = NA) OR (midcum#94333874 = null)) THEN null ELSE cast(midcum#94333874 as float) END AS midcum#94333986, CASE WHEN ((midcumnet#94333875 = NA) OR (midcumnet#94333875 = null)) THEN null ELSE cast(midcumnet#94333875 as float) END AS midcumnet#94333987, CASE WHEN ((midret#94333876 = NA) OR (midret#94333876 = null)) THEN null ELSE cast(midret#94333876 as float) END AS midret#94333988, CASE WHEN ((midretnet#94333877 = NA) OR (midretnet#94333877 = null)) THEN null ELSE cast(midretnet#94333877 as float) END AS midretnet#94334011, CASE WHEN ((smallcum#94333878 = NA) OR (smallcum#94333878 = null)) THEN null ELSE cast(smallcum#94333878 as float) END AS smallcum#94334017, CASE WHEN ((smallcumnet#94333879 = NA) OR (smallcumnet#94333879 = null)) THEN null ELSE cast(smallcumnet#94333879 as float) END AS smallcumnet#94334045, CASE WHEN ((smallret#94333880 = NA) OR (smallret#94333880 = null)) THEN null ELSE cast(smallret#94333880 as float) END AS smallret#94334051, CASE WHEN ((smallretnet#94333881 = NA) OR (smallretnet#94333881 = null)) THEN null ELSE cast(smallretnet#94333881 as float) END AS smallretnet#94334110] +- FileScan csv [date#94333860,allret#94333861,allcum#94333862,drawdown_gross#94333863,drawdown_length_gross#94333864,allretnet#94333865,allcumnet#94333866,drawdown_net#94333867,drawdown_length_net#94333868,numcos#94333869,largecum#94333870,largecumnet#94333871,largeret#94333872,largeretnet#94333873,midcum#94333874,midcumnet#94333875,midret#94333876,midretnet#94333877,smallcum#94333878,smallcumnet#94333879,smallret#94333880,smallretnet#94333881] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94333860, allret#94333861, allcum#94333862, drawdown_gross#94333863, drawdown_length_gross#94333864, allretnet#94333865, allcumnet#94333866, drawdown_net#94333867, drawdown_length_net#94333868, numcos#94333869, largecum#94333870, largecumnet#94333871, largeret#94333872, largeretnet#94333873, midcum#94333874, midcumnet#94333875, midret#94333876, midretnet#94333877, smallcum#94333878, smallcumnet#94333879, smallret#94333880, smallretnet#94333881] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/estimize_signal_history/estimizesignal_preearnings/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94333860 = null) THEN null ELSE cast(date#94333860 as date) END AS date#94333951, CASE WHEN ((allret#94333861 = NA) OR (allret#94333861 = null)) THEN null ELSE cast(allret#94333861 as float) END AS allret#94333952, CASE WHEN ((allcum#94333862 = NA) OR (allcum#94333862 = null)) THEN null ELSE cast(allcum#94333862 as float) END AS allcum#94333953, CASE WHEN ((drawdown_gross#94333863 = NA) OR (drawdown_gross#94333863 = null)) THEN null ELSE cast(drawdown_gross#94333863 as float) END AS drawdown_gross#94333954, CASE WHEN ((drawdown_length_gross#94333864 = NA) OR (drawdown_length_gross#94333864 = null)) THEN null ELSE cast(drawdown_length_gross#94333864 as int) END AS drawdown_length_gross#94333955, CASE WHEN ((allretnet#94333865 = NA) OR (allretnet#94333865 = null)) THEN null ELSE cast(allretnet#94333865 as float) END AS allretnet#94333956, CASE WHEN ((allcumnet#94333866 = NA) OR (allcumnet#94333866 = null)) THEN null ELSE cast(allcumnet#94333866 as float) END AS allcumnet#94333957, CASE WHEN ((drawdown_net#94333867 = NA) OR (drawdown_net#94333867 = null)) THEN null ELSE cast(drawdown_net#94333867 as float) END AS drawdown_net#94333958, CASE WHEN ((drawdown_length_net#94333868 = NA) OR (drawdown_length_net#94333868 = null)) THEN null ELSE cast(drawdown_length_net#94333868 as int) END AS drawdown_length_net#94333964, CASE WHEN ((numcos#94333869 = NA) OR (numcos#94333869 = null)) THEN null ELSE cast(numcos#94333869 as float) END AS numcos#94333970, CASE WHEN ((largecum#94333870 = NA) OR (largecum#94333870 = null)) THEN null ELSE cast(largecum#94333870 as float) END AS largecum#94333976, CASE WHEN ((largecumnet#94333871 = NA) OR (largecumnet#94333871 = null)) THEN null ELSE cast(largecumnet#94333871 as float) END AS largecumnet#94333983, CASE WHEN ((largeret#94333872 = NA) OR (largeret#94333872 = null)) THEN null ELSE cast(largeret#94333872 as float) END AS largeret#94333984, CASE WHEN ((largeretnet#94333873 = NA) OR (largeretnet#94333873 = null)) THEN null ELSE cast(largeretnet#94333873 as float) END AS largeretnet#94333985, CASE WHEN ((midcum#94333874 = NA) OR (midcum#94333874 = null)) THEN null ELSE cast(midcum#94333874 as float) END AS midcum#94333986, CASE WHEN ((midcumnet#94333875 = NA) OR (midcumnet#94333875 = null)) THEN null ELSE cast(midcumnet#94333875 as float) END AS midcumnet#94333987, CASE WHEN ((midret#94333876 = NA) OR (midret#94333876 = null)) THEN null ELSE cast(midret#94333876 as float) END AS midret#94333988, CASE WHEN ((midretnet#94333877 = NA) OR (midretnet#94333877 = null)) THEN null ELSE cast(midretnet#94333877 as float) END AS midretnet#94334011, CASE WHEN ((smallcum#94333878 = NA) OR (smallcum#94333878 = null)) THEN null ELSE cast(smallcum#94333878 as float) END AS smallcum#94334017, CASE WHEN ((smallcumnet#94333879 = NA) OR (smallcumnet#94333879 = null)) THEN null ELSE cast(smallcumnet#94333879 as float) END AS smallcumnet#94334045, CASE WHEN ((smallret#94333880 = NA) OR (smallret#94333880 = null)) THEN null ELSE cast(smallret#94333880 as float) END AS smallret#94334051, CASE WHEN ((smallretnet#94333881 = NA) OR (smallretnet#94333881 = null)) THEN null ELSE cast(smallretnet#94333881 as float) END AS smallretnet#94334110] Input [22]: [date#94333860, allret#94333861, allcum#94333862, drawdown_gross#94333863, drawdown_length_gross#94333864, allretnet#94333865, allcumnet#94333866, drawdown_net#94333867, drawdown_length_net#94333868, numcos#94333869, largecum#94333870, largecumnet#94333871, largeret#94333872, largeretnet#94333873, midcum#94333874, midcumnet#94333875, midret#94333876, midretnet#94333877, smallcum#94333878, smallcumnet#94333879, smallret#94333880, smallretnet#94333881] (7) Project [codegen id : 1] Output [3]: [date#94334879, allret#94334882 AS daily_gross#94335166, allretnet#94334892 AS daily_net#94335167] Input [3]: [allret#94334882, allretnet#94334892, date#94334879] (8) Exchange Input [3]: [date#94334879, daily_gross#94335166, daily_net#94335167] Arguments: rangepartitioning(date#94334879 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7531921] (9) Sort [codegen id : 2] Input [3]: [date#94334879, daily_gross#94335166, daily_net#94335167] Arguments: [date#94334879 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [3]: [date#94334879, daily_gross#94335166, daily_net#94335167] Arguments: 1000000