== Physical Plan ==
CollectLimit (9)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Sort (8)
+- Exchange (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [3]: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
Arguments: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
(2) InMemoryRelation
Arguments: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Sort [date#94390995 ASC NULLS FIRST], true, 0
+- Exchange rangepartitioning(date#94390995 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7536495]
+- InMemoryTableScan [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
+- InMemoryRelation [date#94390995, allret#94390996, allcum#94390997, drawdown_gross#94391008, drawdown_length_gross#94391010, allretnet#94391011, allcumnet#94391101, drawdown_net#94391102, drawdown_length_net#94391103, numcos#94391104, largecum#94391171, largecumnet#94391173, largeret#94391179, largeretnet#94391180, midcum#94391211, midcumnet#94391217, midret#94391219, midretnet#94391220, smallcum#94391226, smallcumnet#94391247, smallret#94391249, smallretnet#94391251], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN (date#94390885 = null) THEN null ELSE cast(date#94390885 as date) END AS date#94390995, CASE WHEN ((allret#94390886 = NA) OR (allret#94390886 = null)) THEN null ELSE cast(allret#94390886 as float) END AS allret#94390996, CASE WHEN ((allcum#94390887 = NA) OR (allcum#94390887 = null)) THEN null ELSE cast(allcum#94390887 as float) END AS allcum#94390997, CASE WHEN ((drawdown_gross#94390888 = NA) OR (drawdown_gross#94390888 = null)) THEN null ELSE cast(drawdown_gross#94390888 as float) END AS drawdown_gross#94391008, CASE WHEN ((drawdown_length_gross#94390889 = NA) OR (drawdown_length_gross#94390889 = null)) THEN null ELSE cast(drawdown_length_gross#94390889 as int) END AS drawdown_length_gross#94391010, CASE WHEN ((allretnet#94390890 = NA) OR (allretnet#94390890 = null)) THEN null ELSE cast(allretnet#94390890 as float) END AS allretnet#94391011, CASE WHEN ((allcumnet#94390891 = NA) OR (allcumnet#94390891 = null)) THEN null ELSE cast(allcumnet#94390891 as float) END AS allcumnet#94391101, CASE WHEN ((drawdown_net#94390892 = NA) OR (drawdown_net#94390892 = null)) THEN null ELSE cast(drawdown_net#94390892 as float) END AS drawdown_net#94391102, CASE WHEN ((drawdown_length_net#94390893 = NA) OR (drawdown_length_net#94390893 = null)) THEN null ELSE cast(drawdown_length_net#94390893 as int) END AS drawdown_length_net#94391103, CASE WHEN ((numcos#94390894 = NA) OR (numcos#94390894 = null)) THEN null ELSE cast(numcos#94390894 as float) END AS numcos#94391104, CASE WHEN ((largecum#94390895 = NA) OR (largecum#94390895 = null)) THEN null ELSE cast(largecum#94390895 as float) END AS largecum#94391171, CASE WHEN ((largecumnet#94390896 = NA) OR (largecumnet#94390896 = null)) THEN null ELSE cast(largecumnet#94390896 as float) END AS largecumnet#94391173, CASE WHEN ((largeret#94390897 = NA) OR (largeret#94390897 = null)) THEN null ELSE cast(largeret#94390897 as float) END AS largeret#94391179, CASE WHEN ((largeretnet#94390898 = NA) OR (largeretnet#94390898 = null)) THEN null ELSE cast(largeretnet#94390898 as float) END AS largeretnet#94391180, CASE WHEN ((midcum#94390899 = NA) OR (midcum#94390899 = null)) THEN null ELSE cast(midcum#94390899 as float) END AS midcum#94391211, CASE WHEN ((midcumnet#94390900 = NA) OR (midcumnet#94390900 = null)) THEN null ELSE cast(midcumnet#94390900 as float) END AS midcumnet#94391217, CASE WHEN ((midret#94390901 = NA) OR (midret#94390901 = null)) THEN null ELSE cast(midret#94390901 as float) END AS midret#94391219, CASE WHEN ((midretnet#94390902 = NA) OR (midretnet#94390902 = null)) THEN null ELSE cast(midretnet#94390902 as float) END AS midretnet#94391220, CASE WHEN ((smallcum#94390903 = NA) OR (smallcum#94390903 = null)) THEN null ELSE cast(smallcum#94390903 as float) END AS smallcum#94391226, CASE WHEN ((smallcumnet#94390904 = NA) OR (smallcumnet#94390904 = null)) THEN null ELSE cast(smallcumnet#94390904 as float) END AS smallcumnet#94391247, CASE WHEN ((smallret#94390905 = NA) OR (smallret#94390905 = null)) THEN null ELSE cast(smallret#94390905 as float) END AS smallret#94391249, CASE WHEN ((smallretnet#94390906 = NA) OR (smallretnet#94390906 = null)) THEN null ELSE cast(smallretnet#94390906 as float) END AS smallretnet#94391251]
+- FileScan csv [date#94390885,allret#94390886,allcum#94390887,drawdown_gross#94390888,drawdown_length_gross#94390889,allretnet#94390890,allcumnet#94390891,drawdown_net#94390892,drawdown_length_net#94390893,numcos#94390894,largecum#94390895,largecumnet#94390896,largeret#94390897,largeretnet#94390898,midcum#94390899,midcumnet#94390900,midret#94390901,midretnet#94390902,smallcum#94390903,smallcumnet#94390904,smallret#94390905,smallretnet#94390906] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None), [date#94390995 ASC NULLS FIRST]
(3) InMemoryTableScan
Output [3]: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
Arguments: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
(4) InMemoryRelation
Arguments: [date#94390995, allret#94390996, allcum#94390997, drawdown_gross#94391008, drawdown_length_gross#94391010, allretnet#94391011, allcumnet#94391101, drawdown_net#94391102, drawdown_length_net#94391103, numcos#94391104, largecum#94391171, largecumnet#94391173, largeret#94391179, largeretnet#94391180, midcum#94391211, midcumnet#94391217, midret#94391219, midretnet#94391220, smallcum#94391226, smallcumnet#94391247, smallret#94391249, smallretnet#94391251], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94390885 = null) THEN null ELSE cast(date#94390885 as date) END AS date#94390995, CASE WHEN ((allret#94390886 = NA) OR (allret#94390886 = null)) THEN null ELSE cast(allret#94390886 as float) END AS allret#94390996, CASE WHEN ((allcum#94390887 = NA) OR (allcum#94390887 = null)) THEN null ELSE cast(allcum#94390887 as float) END AS allcum#94390997, CASE WHEN ((drawdown_gross#94390888 = NA) OR (drawdown_gross#94390888 = null)) THEN null ELSE cast(drawdown_gross#94390888 as float) END AS drawdown_gross#94391008, CASE WHEN ((drawdown_length_gross#94390889 = NA) OR (drawdown_length_gross#94390889 = null)) THEN null ELSE cast(drawdown_length_gross#94390889 as int) END AS drawdown_length_gross#94391010, CASE WHEN ((allretnet#94390890 = NA) OR (allretnet#94390890 = null)) THEN null ELSE cast(allretnet#94390890 as float) END AS allretnet#94391011, CASE WHEN ((allcumnet#94390891 = NA) OR (allcumnet#94390891 = null)) THEN null ELSE cast(allcumnet#94390891 as float) END AS allcumnet#94391101, CASE WHEN ((drawdown_net#94390892 = NA) OR (drawdown_net#94390892 = null)) THEN null ELSE cast(drawdown_net#94390892 as float) END AS drawdown_net#94391102, CASE WHEN ((drawdown_length_net#94390893 = NA) OR (drawdown_length_net#94390893 = null)) THEN null ELSE cast(drawdown_length_net#94390893 as int) END AS drawdown_length_net#94391103, CASE WHEN ((numcos#94390894 = NA) OR (numcos#94390894 = null)) THEN null ELSE cast(numcos#94390894 as float) END AS numcos#94391104, CASE WHEN ((largecum#94390895 = NA) OR (largecum#94390895 = null)) THEN null ELSE cast(largecum#94390895 as float) END AS largecum#94391171, CASE WHEN ((largecumnet#94390896 = NA) OR (largecumnet#94390896 = null)) THEN null ELSE cast(largecumnet#94390896 as float) END AS largecumnet#94391173, CASE WHEN ((largeret#94390897 = NA) OR (largeret#94390897 = null)) THEN null ELSE cast(largeret#94390897 as float) END AS largeret#94391179, CASE WHEN ((largeretnet#94390898 = NA) OR (largeretnet#94390898 = null)) THEN null ELSE cast(largeretnet#94390898 as float) END AS largeretnet#94391180, CASE WHEN ((midcum#94390899 = NA) OR (midcum#94390899 = null)) THEN null ELSE cast(midcum#94390899 as float) END AS midcum#94391211, CASE WHEN ((midcumnet#94390900 = NA) OR (midcumnet#94390900 = null)) THEN null ELSE cast(midcumnet#94390900 as float) END AS midcumnet#94391217, CASE WHEN ((midret#94390901 = NA) OR (midret#94390901 = null)) THEN null ELSE cast(midret#94390901 as float) END AS midret#94391219, CASE WHEN ((midretnet#94390902 = NA) OR (midretnet#94390902 = null)) THEN null ELSE cast(midretnet#94390902 as float) END AS midretnet#94391220, CASE WHEN ((smallcum#94390903 = NA) OR (smallcum#94390903 = null)) THEN null ELSE cast(smallcum#94390903 as float) END AS smallcum#94391226, CASE WHEN ((smallcumnet#94390904 = NA) OR (smallcumnet#94390904 = null)) THEN null ELSE cast(smallcumnet#94390904 as float) END AS smallcumnet#94391247, CASE WHEN ((smallret#94390905 = NA) OR (smallret#94390905 = null)) THEN null ELSE cast(smallret#94390905 as float) END AS smallret#94391249, CASE WHEN ((smallretnet#94390906 = NA) OR (smallretnet#94390906 = null)) THEN null ELSE cast(smallretnet#94390906 as float) END AS smallretnet#94391251]
+- FileScan csv [date#94390885,allret#94390886,allcum#94390887,drawdown_gross#94390888,drawdown_length_gross#94390889,allretnet#94390890,allcumnet#94390891,drawdown_net#94390892,drawdown_length_net#94390893,numcos#94390894,largecum#94390895,largecumnet#94390896,largeret#94390897,largeretnet#94390898,midcum#94390899,midcumnet#94390900,midret#94390901,midretnet#94390902,smallcum#94390903,smallcumnet#94390904,smallret#94390905,smallretnet#94390906] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None)
(5) Scan csv
Output [22]: [date#94390885, allret#94390886, allcum#94390887, drawdown_gross#94390888, drawdown_length_gross#94390889, allretnet#94390890, allcumnet#94390891, drawdown_net#94390892, drawdown_length_net#94390893, numcos#94390894, largecum#94390895, largecumnet#94390896, largeret#94390897, largeretnet#94390898, midcum#94390899, midcumnet#94390900, midret#94390901, midretnet#94390902, smallcum#94390903, smallcumnet#94390904, smallret#94390905, smallretnet#94390906]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/REPORT_cumret.csv]
ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string>
(6) Project [codegen id : 1]
Output [22]: [CASE WHEN (date#94390885 = null) THEN null ELSE cast(date#94390885 as date) END AS date#94390995, CASE WHEN ((allret#94390886 = NA) OR (allret#94390886 = null)) THEN null ELSE cast(allret#94390886 as float) END AS allret#94390996, CASE WHEN ((allcum#94390887 = NA) OR (allcum#94390887 = null)) THEN null ELSE cast(allcum#94390887 as float) END AS allcum#94390997, CASE WHEN ((drawdown_gross#94390888 = NA) OR (drawdown_gross#94390888 = null)) THEN null ELSE cast(drawdown_gross#94390888 as float) END AS drawdown_gross#94391008, CASE WHEN ((drawdown_length_gross#94390889 = NA) OR (drawdown_length_gross#94390889 = null)) THEN null ELSE cast(drawdown_length_gross#94390889 as int) END AS drawdown_length_gross#94391010, CASE WHEN ((allretnet#94390890 = NA) OR (allretnet#94390890 = null)) THEN null ELSE cast(allretnet#94390890 as float) END AS allretnet#94391011, CASE WHEN ((allcumnet#94390891 = NA) OR (allcumnet#94390891 = null)) THEN null ELSE cast(allcumnet#94390891 as float) END AS allcumnet#94391101, CASE WHEN ((drawdown_net#94390892 = NA) OR (drawdown_net#94390892 = null)) THEN null ELSE cast(drawdown_net#94390892 as float) END AS drawdown_net#94391102, CASE WHEN ((drawdown_length_net#94390893 = NA) OR (drawdown_length_net#94390893 = null)) THEN null ELSE cast(drawdown_length_net#94390893 as int) END AS drawdown_length_net#94391103, CASE WHEN ((numcos#94390894 = NA) OR (numcos#94390894 = null)) THEN null ELSE cast(numcos#94390894 as float) END AS numcos#94391104, CASE WHEN ((largecum#94390895 = NA) OR (largecum#94390895 = null)) THEN null ELSE cast(largecum#94390895 as float) END AS largecum#94391171, CASE WHEN ((largecumnet#94390896 = NA) OR (largecumnet#94390896 = null)) THEN null ELSE cast(largecumnet#94390896 as float) END AS largecumnet#94391173, CASE WHEN ((largeret#94390897 = NA) OR (largeret#94390897 = null)) THEN null ELSE cast(largeret#94390897 as float) END AS largeret#94391179, CASE WHEN ((largeretnet#94390898 = NA) OR (largeretnet#94390898 = null)) THEN null ELSE cast(largeretnet#94390898 as float) END AS largeretnet#94391180, CASE WHEN ((midcum#94390899 = NA) OR (midcum#94390899 = null)) THEN null ELSE cast(midcum#94390899 as float) END AS midcum#94391211, CASE WHEN ((midcumnet#94390900 = NA) OR (midcumnet#94390900 = null)) THEN null ELSE cast(midcumnet#94390900 as float) END AS midcumnet#94391217, CASE WHEN ((midret#94390901 = NA) OR (midret#94390901 = null)) THEN null ELSE cast(midret#94390901 as float) END AS midret#94391219, CASE WHEN ((midretnet#94390902 = NA) OR (midretnet#94390902 = null)) THEN null ELSE cast(midretnet#94390902 as float) END AS midretnet#94391220, CASE WHEN ((smallcum#94390903 = NA) OR (smallcum#94390903 = null)) THEN null ELSE cast(smallcum#94390903 as float) END AS smallcum#94391226, CASE WHEN ((smallcumnet#94390904 = NA) OR (smallcumnet#94390904 = null)) THEN null ELSE cast(smallcumnet#94390904 as float) END AS smallcumnet#94391247, CASE WHEN ((smallret#94390905 = NA) OR (smallret#94390905 = null)) THEN null ELSE cast(smallret#94390905 as float) END AS smallret#94391249, CASE WHEN ((smallretnet#94390906 = NA) OR (smallretnet#94390906 = null)) THEN null ELSE cast(smallretnet#94390906 as float) END AS smallretnet#94391251]
Input [22]: [date#94390885, allret#94390886, allcum#94390887, drawdown_gross#94390888, drawdown_length_gross#94390889, allretnet#94390890, allcumnet#94390891, drawdown_net#94390892, drawdown_length_net#94390893, numcos#94390894, largecum#94390895, largecumnet#94390896, largeret#94390897, largeretnet#94390898, midcum#94390899, midcumnet#94390900, midret#94390901, midretnet#94390902, smallcum#94390903, smallcumnet#94390904, smallret#94390905, smallretnet#94390906]
(7) Exchange
Input [3]: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
Arguments: rangepartitioning(date#94390995 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7536495]
(8) Sort [codegen id : 1]
Input [3]: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
Arguments: [date#94390995 ASC NULLS FIRST], true, 0
(9) CollectLimit
Input [3]: [date#94390995, drawdown_gross#94391008, drawdown_net#94391102]
Arguments: 1000000