== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [date#94392013, daily_gross#94392261, daily_net#94392262] Arguments: [date#94392013, daily_gross#94392261, daily_net#94392262] (2) InMemoryRelation Arguments: [date#94392013, daily_gross#94392261, daily_net#94392262], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94392013 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94392013 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7536477] +- *(1) Project [date#94392013, allret#94392015 AS daily_gross#94392261, allretnet#94392022 AS daily_net#94392262] +- InMemoryTableScan [allret#94392015, allretnet#94392022, date#94392013] +- InMemoryRelation [date#94392013, allret#94392015, allcum#94392017, drawdown_gross#94392019, drawdown_length_gross#94392021, allretnet#94392022, allcumnet#94392024, drawdown_net#94392026, drawdown_length_net#94392029, numcos#94392031, largecum#94392033, largecumnet#94392058, largeret#94392083, largeretnet#94392084, midcum#94392086, midcumnet#94392111, midret#94392136, midretnet#94392138, smallcum#94392139, smallcumnet#94392141, smallret#94392143, smallretnet#94392145], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94390885 = null) THEN null ELSE cast(date#94390885 as date) END AS date#94390995, CASE WHEN ((allret#94390886 = NA) OR (allret#94390886 = null)) THEN null ELSE cast(allret#94390886 as float) END AS allret#94390996, CASE WHEN ((allcum#94390887 = NA) OR (allcum#94390887 = null)) THEN null ELSE cast(allcum#94390887 as float) END AS allcum#94390997, CASE WHEN ((drawdown_gross#94390888 = NA) OR (drawdown_gross#94390888 = null)) THEN null ELSE cast(drawdown_gross#94390888 as float) END AS drawdown_gross#94391008, CASE WHEN ((drawdown_length_gross#94390889 = NA) OR (drawdown_length_gross#94390889 = null)) THEN null ELSE cast(drawdown_length_gross#94390889 as int) END AS drawdown_length_gross#94391010, CASE WHEN ((allretnet#94390890 = NA) OR (allretnet#94390890 = null)) THEN null ELSE cast(allretnet#94390890 as float) END AS allretnet#94391011, CASE WHEN ((allcumnet#94390891 = NA) OR (allcumnet#94390891 = null)) THEN null ELSE cast(allcumnet#94390891 as float) END AS allcumnet#94391101, CASE WHEN ((drawdown_net#94390892 = NA) OR (drawdown_net#94390892 = null)) THEN null ELSE cast(drawdown_net#94390892 as float) END AS drawdown_net#94391102, CASE WHEN ((drawdown_length_net#94390893 = NA) OR (drawdown_length_net#94390893 = null)) THEN null ELSE cast(drawdown_length_net#94390893 as int) END AS drawdown_length_net#94391103, CASE WHEN ((numcos#94390894 = NA) OR (numcos#94390894 = null)) THEN null ELSE cast(numcos#94390894 as float) END AS numcos#94391104, CASE WHEN ((largecum#94390895 = NA) OR (largecum#94390895 = null)) THEN null ELSE cast(largecum#94390895 as float) END AS largecum#94391171, CASE WHEN ((largecumnet#94390896 = NA) OR (largecumnet#94390896 = null)) THEN null ELSE cast(largecumnet#94390896 as float) END AS largecumnet#94391173, CASE WHEN ((largeret#94390897 = NA) OR (largeret#94390897 = null)) THEN null ELSE cast(largeret#94390897 as float) END AS largeret#94391179, CASE WHEN ((largeretnet#94390898 = NA) OR (largeretnet#94390898 = null)) THEN null ELSE cast(largeretnet#94390898 as float) END AS largeretnet#94391180, CASE WHEN ((midcum#94390899 = NA) OR (midcum#94390899 = null)) THEN null ELSE cast(midcum#94390899 as float) END AS midcum#94391211, CASE WHEN ((midcumnet#94390900 = NA) OR (midcumnet#94390900 = null)) THEN null ELSE cast(midcumnet#94390900 as float) END AS midcumnet#94391217, CASE WHEN ((midret#94390901 = NA) OR (midret#94390901 = null)) THEN null ELSE cast(midret#94390901 as float) END AS midret#94391219, CASE WHEN ((midretnet#94390902 = NA) OR (midretnet#94390902 = null)) THEN null ELSE cast(midretnet#94390902 as float) END AS midretnet#94391220, CASE WHEN ((smallcum#94390903 = NA) OR (smallcum#94390903 = null)) THEN null ELSE cast(smallcum#94390903 as float) END AS smallcum#94391226, CASE WHEN ((smallcumnet#94390904 = NA) OR (smallcumnet#94390904 = null)) THEN null ELSE cast(smallcumnet#94390904 as float) END AS smallcumnet#94391247, CASE WHEN ((smallret#94390905 = NA) OR (smallret#94390905 = null)) THEN null ELSE cast(smallret#94390905 as float) END AS smallret#94391249, CASE WHEN ((smallretnet#94390906 = NA) OR (smallretnet#94390906 = null)) THEN null ELSE cast(smallretnet#94390906 as float) END AS smallretnet#94391251] +- FileScan csv [date#94390885,allret#94390886,allcum#94390887,drawdown_gross#94390888,drawdown_length_gross#94390889,allretnet#94390890,allcumnet#94390891,drawdown_net#94390892,drawdown_length_net#94390893,numcos#94390894,largecum#94390895,largecumnet#94390896,largeret#94390897,largeretnet#94390898,midcum#94390899,midcumnet#94390900,midret#94390901,midretnet#94390902,smallcum#94390903,smallcumnet#94390904,smallret#94390905,smallretnet#94390906] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94392013 ASC NULLS FIRST] (3) InMemoryTableScan Output [3]: [allret#94392015, allretnet#94392022, date#94392013] Arguments: [allret#94392015, allretnet#94392022, date#94392013] (4) InMemoryRelation Arguments: [date#94392013, allret#94392015, allcum#94392017, drawdown_gross#94392019, drawdown_length_gross#94392021, allretnet#94392022, allcumnet#94392024, drawdown_net#94392026, drawdown_length_net#94392029, numcos#94392031, largecum#94392033, largecumnet#94392058, largeret#94392083, largeretnet#94392084, midcum#94392086, midcumnet#94392111, midret#94392136, midretnet#94392138, smallcum#94392139, smallcumnet#94392141, smallret#94392143, smallretnet#94392145], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94390885 = null) THEN null ELSE cast(date#94390885 as date) END AS date#94390995, CASE WHEN ((allret#94390886 = NA) OR (allret#94390886 = null)) THEN null ELSE cast(allret#94390886 as float) END AS allret#94390996, CASE WHEN ((allcum#94390887 = NA) OR (allcum#94390887 = null)) THEN null ELSE cast(allcum#94390887 as float) END AS allcum#94390997, CASE WHEN ((drawdown_gross#94390888 = NA) OR (drawdown_gross#94390888 = null)) THEN null ELSE cast(drawdown_gross#94390888 as float) END AS drawdown_gross#94391008, CASE WHEN ((drawdown_length_gross#94390889 = NA) OR (drawdown_length_gross#94390889 = null)) THEN null ELSE cast(drawdown_length_gross#94390889 as int) END AS drawdown_length_gross#94391010, CASE WHEN ((allretnet#94390890 = NA) OR (allretnet#94390890 = null)) THEN null ELSE cast(allretnet#94390890 as float) END AS allretnet#94391011, CASE WHEN ((allcumnet#94390891 = NA) OR (allcumnet#94390891 = null)) THEN null ELSE cast(allcumnet#94390891 as float) END AS allcumnet#94391101, CASE WHEN ((drawdown_net#94390892 = NA) OR (drawdown_net#94390892 = null)) THEN null ELSE cast(drawdown_net#94390892 as float) END AS drawdown_net#94391102, CASE WHEN ((drawdown_length_net#94390893 = NA) OR (drawdown_length_net#94390893 = null)) THEN null ELSE cast(drawdown_length_net#94390893 as int) END AS drawdown_length_net#94391103, CASE WHEN ((numcos#94390894 = NA) OR (numcos#94390894 = null)) THEN null ELSE cast(numcos#94390894 as float) END AS numcos#94391104, CASE WHEN ((largecum#94390895 = NA) OR (largecum#94390895 = null)) THEN null ELSE cast(largecum#94390895 as float) END AS largecum#94391171, CASE WHEN ((largecumnet#94390896 = NA) OR (largecumnet#94390896 = null)) THEN null ELSE cast(largecumnet#94390896 as float) END AS largecumnet#94391173, CASE WHEN ((largeret#94390897 = NA) OR (largeret#94390897 = null)) THEN null ELSE cast(largeret#94390897 as float) END AS largeret#94391179, CASE WHEN ((largeretnet#94390898 = NA) OR (largeretnet#94390898 = null)) THEN null ELSE cast(largeretnet#94390898 as float) END AS largeretnet#94391180, CASE WHEN ((midcum#94390899 = NA) OR (midcum#94390899 = null)) THEN null ELSE cast(midcum#94390899 as float) END AS midcum#94391211, CASE WHEN ((midcumnet#94390900 = NA) OR (midcumnet#94390900 = null)) THEN null ELSE cast(midcumnet#94390900 as float) END AS midcumnet#94391217, CASE WHEN ((midret#94390901 = NA) OR (midret#94390901 = null)) THEN null ELSE cast(midret#94390901 as float) END AS midret#94391219, CASE WHEN ((midretnet#94390902 = NA) OR (midretnet#94390902 = null)) THEN null ELSE cast(midretnet#94390902 as float) END AS midretnet#94391220, CASE WHEN ((smallcum#94390903 = NA) OR (smallcum#94390903 = null)) THEN null ELSE cast(smallcum#94390903 as float) END AS smallcum#94391226, CASE WHEN ((smallcumnet#94390904 = NA) OR (smallcumnet#94390904 = null)) THEN null ELSE cast(smallcumnet#94390904 as float) END AS smallcumnet#94391247, CASE WHEN ((smallret#94390905 = NA) OR (smallret#94390905 = null)) THEN null ELSE cast(smallret#94390905 as float) END AS smallret#94391249, CASE WHEN ((smallretnet#94390906 = NA) OR (smallretnet#94390906 = null)) THEN null ELSE cast(smallretnet#94390906 as float) END AS smallretnet#94391251] +- FileScan csv [date#94390885,allret#94390886,allcum#94390887,drawdown_gross#94390888,drawdown_length_gross#94390889,allretnet#94390890,allcumnet#94390891,drawdown_net#94390892,drawdown_length_net#94390893,numcos#94390894,largecum#94390895,largecumnet#94390896,largeret#94390897,largeretnet#94390898,midcum#94390899,midcumnet#94390900,midret#94390901,midretnet#94390902,smallcum#94390903,smallcumnet#94390904,smallret#94390905,smallretnet#94390906] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94390885, allret#94390886, allcum#94390887, drawdown_gross#94390888, drawdown_length_gross#94390889, allretnet#94390890, allcumnet#94390891, drawdown_net#94390892, drawdown_length_net#94390893, numcos#94390894, largecum#94390895, largecumnet#94390896, largeret#94390897, largeretnet#94390898, midcum#94390899, midcumnet#94390900, midret#94390901, midretnet#94390902, smallcum#94390903, smallcumnet#94390904, smallret#94390905, smallretnet#94390906] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94390885 = null) THEN null ELSE cast(date#94390885 as date) END AS date#94390995, CASE WHEN ((allret#94390886 = NA) OR (allret#94390886 = null)) THEN null ELSE cast(allret#94390886 as float) END AS allret#94390996, CASE WHEN ((allcum#94390887 = NA) OR (allcum#94390887 = null)) THEN null ELSE cast(allcum#94390887 as float) END AS allcum#94390997, CASE WHEN ((drawdown_gross#94390888 = NA) OR (drawdown_gross#94390888 = null)) THEN null ELSE cast(drawdown_gross#94390888 as float) END AS drawdown_gross#94391008, CASE WHEN ((drawdown_length_gross#94390889 = NA) OR (drawdown_length_gross#94390889 = null)) THEN null ELSE cast(drawdown_length_gross#94390889 as int) END AS drawdown_length_gross#94391010, CASE WHEN ((allretnet#94390890 = NA) OR (allretnet#94390890 = null)) THEN null ELSE cast(allretnet#94390890 as float) END AS allretnet#94391011, CASE WHEN ((allcumnet#94390891 = NA) OR (allcumnet#94390891 = null)) THEN null ELSE cast(allcumnet#94390891 as float) END AS allcumnet#94391101, CASE WHEN ((drawdown_net#94390892 = NA) OR (drawdown_net#94390892 = null)) THEN null ELSE cast(drawdown_net#94390892 as float) END AS drawdown_net#94391102, CASE WHEN ((drawdown_length_net#94390893 = NA) OR (drawdown_length_net#94390893 = null)) THEN null ELSE cast(drawdown_length_net#94390893 as int) END AS drawdown_length_net#94391103, CASE WHEN ((numcos#94390894 = NA) OR (numcos#94390894 = null)) THEN null ELSE cast(numcos#94390894 as float) END AS numcos#94391104, CASE WHEN ((largecum#94390895 = NA) OR (largecum#94390895 = null)) THEN null ELSE cast(largecum#94390895 as float) END AS largecum#94391171, CASE WHEN ((largecumnet#94390896 = NA) OR (largecumnet#94390896 = null)) THEN null ELSE cast(largecumnet#94390896 as float) END AS largecumnet#94391173, CASE WHEN ((largeret#94390897 = NA) OR (largeret#94390897 = null)) THEN null ELSE cast(largeret#94390897 as float) END AS largeret#94391179, CASE WHEN ((largeretnet#94390898 = NA) OR (largeretnet#94390898 = null)) THEN null ELSE cast(largeretnet#94390898 as float) END AS largeretnet#94391180, CASE WHEN ((midcum#94390899 = NA) OR (midcum#94390899 = null)) THEN null ELSE cast(midcum#94390899 as float) END AS midcum#94391211, CASE WHEN ((midcumnet#94390900 = NA) OR (midcumnet#94390900 = null)) THEN null ELSE cast(midcumnet#94390900 as float) END AS midcumnet#94391217, CASE WHEN ((midret#94390901 = NA) OR (midret#94390901 = null)) THEN null ELSE cast(midret#94390901 as float) END AS midret#94391219, CASE WHEN ((midretnet#94390902 = NA) OR (midretnet#94390902 = null)) THEN null ELSE cast(midretnet#94390902 as float) END AS midretnet#94391220, CASE WHEN ((smallcum#94390903 = NA) OR (smallcum#94390903 = null)) THEN null ELSE cast(smallcum#94390903 as float) END AS smallcum#94391226, CASE WHEN ((smallcumnet#94390904 = NA) OR (smallcumnet#94390904 = null)) THEN null ELSE cast(smallcumnet#94390904 as float) END AS smallcumnet#94391247, CASE WHEN ((smallret#94390905 = NA) OR (smallret#94390905 = null)) THEN null ELSE cast(smallret#94390905 as float) END AS smallret#94391249, CASE WHEN ((smallretnet#94390906 = NA) OR (smallretnet#94390906 = null)) THEN null ELSE cast(smallretnet#94390906 as float) END AS smallretnet#94391251] Input [22]: [date#94390885, allret#94390886, allcum#94390887, drawdown_gross#94390888, drawdown_length_gross#94390889, allretnet#94390890, allcumnet#94390891, drawdown_net#94390892, drawdown_length_net#94390893, numcos#94390894, largecum#94390895, largecumnet#94390896, largeret#94390897, largeretnet#94390898, midcum#94390899, midcumnet#94390900, midret#94390901, midretnet#94390902, smallcum#94390903, smallcumnet#94390904, smallret#94390905, smallretnet#94390906] (7) Project [codegen id : 1] Output [3]: [date#94392013, allret#94392015 AS daily_gross#94392261, allretnet#94392022 AS daily_net#94392262] Input [3]: [allret#94392015, allretnet#94392022, date#94392013] (8) Exchange Input [3]: [date#94392013, daily_gross#94392261, daily_net#94392262] Arguments: rangepartitioning(date#94392013 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7536477] (9) Sort [codegen id : 2] Input [3]: [date#94392013, daily_gross#94392261, daily_net#94392262] Arguments: [date#94392013 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [3]: [date#94392013, daily_gross#94392261, daily_net#94392262] Arguments: 1000000