== Physical Plan ==
CollectLimit (9)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Sort (8)
+- Exchange (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [3]: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
Arguments: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
(2) InMemoryRelation
Arguments: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Sort [date#94305864 ASC NULLS FIRST], true, 0
+- Exchange rangepartitioning(date#94305864 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7529657]
+- InMemoryTableScan [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
+- InMemoryRelation [date#94305864, allret#94305867, allcum#94305869, drawdown_gross#94305870, drawdown_length_gross#94305872, allretnet#94305875, allcumnet#94305877, drawdown_net#94305879, drawdown_length_net#94305881, numcos#94305884, largecum#94305886, largecumnet#94305888, largeret#94305890, largeretnet#94305892, midcum#94305894, midcumnet#94305896, midret#94305898, midretnet#94305899, smallcum#94305902, smallcumnet#94305905, smallret#94305907, smallretnet#94305909], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN (date#94305708 = null) THEN null ELSE cast(date#94305708 as date) END AS date#94305864, CASE WHEN ((allret#94305709 = NA) OR (allret#94305709 = null)) THEN null ELSE cast(allret#94305709 as float) END AS allret#94305867, CASE WHEN ((allcum#94305710 = NA) OR (allcum#94305710 = null)) THEN null ELSE cast(allcum#94305710 as float) END AS allcum#94305869, CASE WHEN ((drawdown_gross#94305711 = NA) OR (drawdown_gross#94305711 = null)) THEN null ELSE cast(drawdown_gross#94305711 as float) END AS drawdown_gross#94305870, CASE WHEN ((drawdown_length_gross#94305712 = NA) OR (drawdown_length_gross#94305712 = null)) THEN null ELSE cast(drawdown_length_gross#94305712 as int) END AS drawdown_length_gross#94305872, CASE WHEN ((allretnet#94305713 = NA) OR (allretnet#94305713 = null)) THEN null ELSE cast(allretnet#94305713 as float) END AS allretnet#94305875, CASE WHEN ((allcumnet#94305714 = NA) OR (allcumnet#94305714 = null)) THEN null ELSE cast(allcumnet#94305714 as float) END AS allcumnet#94305877, CASE WHEN ((drawdown_net#94305715 = NA) OR (drawdown_net#94305715 = null)) THEN null ELSE cast(drawdown_net#94305715 as float) END AS drawdown_net#94305879, CASE WHEN ((drawdown_length_net#94305716 = NA) OR (drawdown_length_net#94305716 = null)) THEN null ELSE cast(drawdown_length_net#94305716 as int) END AS drawdown_length_net#94305881, CASE WHEN ((numcos#94305717 = NA) OR (numcos#94305717 = null)) THEN null ELSE cast(numcos#94305717 as float) END AS numcos#94305884, CASE WHEN ((largecum#94305718 = NA) OR (largecum#94305718 = null)) THEN null ELSE cast(largecum#94305718 as float) END AS largecum#94305886, CASE WHEN ((largecumnet#94305719 = NA) OR (largecumnet#94305719 = null)) THEN null ELSE cast(largecumnet#94305719 as float) END AS largecumnet#94305888, CASE WHEN ((largeret#94305720 = NA) OR (largeret#94305720 = null)) THEN null ELSE cast(largeret#94305720 as float) END AS largeret#94305890, CASE WHEN ((largeretnet#94305721 = NA) OR (largeretnet#94305721 = null)) THEN null ELSE cast(largeretnet#94305721 as float) END AS largeretnet#94305892, CASE WHEN ((midcum#94305722 = NA) OR (midcum#94305722 = null)) THEN null ELSE cast(midcum#94305722 as float) END AS midcum#94305894, CASE WHEN ((midcumnet#94305723 = NA) OR (midcumnet#94305723 = null)) THEN null ELSE cast(midcumnet#94305723 as float) END AS midcumnet#94305896, CASE WHEN ((midret#94305724 = NA) OR (midret#94305724 = null)) THEN null ELSE cast(midret#94305724 as float) END AS midret#94305898, CASE WHEN ((midretnet#94305725 = NA) OR (midretnet#94305725 = null)) THEN null ELSE cast(midretnet#94305725 as float) END AS midretnet#94305899, CASE WHEN ((smallcum#94305726 = NA) OR (smallcum#94305726 = null)) THEN null ELSE cast(smallcum#94305726 as float) END AS smallcum#94305902, CASE WHEN ((smallcumnet#94305727 = NA) OR (smallcumnet#94305727 = null)) THEN null ELSE cast(smallcumnet#94305727 as float) END AS smallcumnet#94305905, CASE WHEN ((smallret#94305728 = NA) OR (smallret#94305728 = null)) THEN null ELSE cast(smallret#94305728 as float) END AS smallret#94305907, CASE WHEN ((smallretnet#94305729 = NA) OR (smallretnet#94305729 = null)) THEN null ELSE cast(smallretnet#94305729 as float) END AS smallretnet#94305909]
+- FileScan csv [date#94305708,allret#94305709,allcum#94305710,drawdown_gross#94305711,drawdown_length_gross#94305712,allretnet#94305713,allcumnet#94305714,drawdown_net#94305715,drawdown_length_net#94305716,numcos#94305717,largecum#94305718,largecumnet#94305719,largeret#94305720,largeretnet#94305721,midcum#94305722,midcumnet#94305723,midret#94305724,midretnet#94305725,smallcum#94305726,smallcumnet#94305727,smallret#94305728,smallretnet#94305729] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None), [date#94305864 ASC NULLS FIRST]
(3) InMemoryTableScan
Output [3]: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
Arguments: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
(4) InMemoryRelation
Arguments: [date#94305864, allret#94305867, allcum#94305869, drawdown_gross#94305870, drawdown_length_gross#94305872, allretnet#94305875, allcumnet#94305877, drawdown_net#94305879, drawdown_length_net#94305881, numcos#94305884, largecum#94305886, largecumnet#94305888, largeret#94305890, largeretnet#94305892, midcum#94305894, midcumnet#94305896, midret#94305898, midretnet#94305899, smallcum#94305902, smallcumnet#94305905, smallret#94305907, smallretnet#94305909], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94305708 = null) THEN null ELSE cast(date#94305708 as date) END AS date#94305864, CASE WHEN ((allret#94305709 = NA) OR (allret#94305709 = null)) THEN null ELSE cast(allret#94305709 as float) END AS allret#94305867, CASE WHEN ((allcum#94305710 = NA) OR (allcum#94305710 = null)) THEN null ELSE cast(allcum#94305710 as float) END AS allcum#94305869, CASE WHEN ((drawdown_gross#94305711 = NA) OR (drawdown_gross#94305711 = null)) THEN null ELSE cast(drawdown_gross#94305711 as float) END AS drawdown_gross#94305870, CASE WHEN ((drawdown_length_gross#94305712 = NA) OR (drawdown_length_gross#94305712 = null)) THEN null ELSE cast(drawdown_length_gross#94305712 as int) END AS drawdown_length_gross#94305872, CASE WHEN ((allretnet#94305713 = NA) OR (allretnet#94305713 = null)) THEN null ELSE cast(allretnet#94305713 as float) END AS allretnet#94305875, CASE WHEN ((allcumnet#94305714 = NA) OR (allcumnet#94305714 = null)) THEN null ELSE cast(allcumnet#94305714 as float) END AS allcumnet#94305877, CASE WHEN ((drawdown_net#94305715 = NA) OR (drawdown_net#94305715 = null)) THEN null ELSE cast(drawdown_net#94305715 as float) END AS drawdown_net#94305879, CASE WHEN ((drawdown_length_net#94305716 = NA) OR (drawdown_length_net#94305716 = null)) THEN null ELSE cast(drawdown_length_net#94305716 as int) END AS drawdown_length_net#94305881, CASE WHEN ((numcos#94305717 = NA) OR (numcos#94305717 = null)) THEN null ELSE cast(numcos#94305717 as float) END AS numcos#94305884, CASE WHEN ((largecum#94305718 = NA) OR (largecum#94305718 = null)) THEN null ELSE cast(largecum#94305718 as float) END AS largecum#94305886, CASE WHEN ((largecumnet#94305719 = NA) OR (largecumnet#94305719 = null)) THEN null ELSE cast(largecumnet#94305719 as float) END AS largecumnet#94305888, CASE WHEN ((largeret#94305720 = NA) OR (largeret#94305720 = null)) THEN null ELSE cast(largeret#94305720 as float) END AS largeret#94305890, CASE WHEN ((largeretnet#94305721 = NA) OR (largeretnet#94305721 = null)) THEN null ELSE cast(largeretnet#94305721 as float) END AS largeretnet#94305892, CASE WHEN ((midcum#94305722 = NA) OR (midcum#94305722 = null)) THEN null ELSE cast(midcum#94305722 as float) END AS midcum#94305894, CASE WHEN ((midcumnet#94305723 = NA) OR (midcumnet#94305723 = null)) THEN null ELSE cast(midcumnet#94305723 as float) END AS midcumnet#94305896, CASE WHEN ((midret#94305724 = NA) OR (midret#94305724 = null)) THEN null ELSE cast(midret#94305724 as float) END AS midret#94305898, CASE WHEN ((midretnet#94305725 = NA) OR (midretnet#94305725 = null)) THEN null ELSE cast(midretnet#94305725 as float) END AS midretnet#94305899, CASE WHEN ((smallcum#94305726 = NA) OR (smallcum#94305726 = null)) THEN null ELSE cast(smallcum#94305726 as float) END AS smallcum#94305902, CASE WHEN ((smallcumnet#94305727 = NA) OR (smallcumnet#94305727 = null)) THEN null ELSE cast(smallcumnet#94305727 as float) END AS smallcumnet#94305905, CASE WHEN ((smallret#94305728 = NA) OR (smallret#94305728 = null)) THEN null ELSE cast(smallret#94305728 as float) END AS smallret#94305907, CASE WHEN ((smallretnet#94305729 = NA) OR (smallretnet#94305729 = null)) THEN null ELSE cast(smallretnet#94305729 as float) END AS smallretnet#94305909]
+- FileScan csv [date#94305708,allret#94305709,allcum#94305710,drawdown_gross#94305711,drawdown_length_gross#94305712,allretnet#94305713,allcumnet#94305714,drawdown_net#94305715,drawdown_length_net#94305716,numcos#94305717,largecum#94305718,largecumnet#94305719,largeret#94305720,largeretnet#94305721,midcum#94305722,midcumnet#94305723,midret#94305724,midretnet#94305725,smallcum#94305726,smallcumnet#94305727,smallret#94305728,smallretnet#94305729] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None)
(5) Scan csv
Output [22]: [date#94305708, allret#94305709, allcum#94305710, drawdown_gross#94305711, drawdown_length_gross#94305712, allretnet#94305713, allcumnet#94305714, drawdown_net#94305715, drawdown_length_net#94305716, numcos#94305717, largecum#94305718, largecumnet#94305719, largeret#94305720, largeretnet#94305721, midcum#94305722, midcumnet#94305723, midret#94305724, midretnet#94305725, smallcum#94305726, smallcumnet#94305727, smallret#94305728, smallretnet#94305729]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/estimize_signal_history/estimizesignal/REPORT_cumret.csv]
ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string>
(6) Project [codegen id : 1]
Output [22]: [CASE WHEN (date#94305708 = null) THEN null ELSE cast(date#94305708 as date) END AS date#94305864, CASE WHEN ((allret#94305709 = NA) OR (allret#94305709 = null)) THEN null ELSE cast(allret#94305709 as float) END AS allret#94305867, CASE WHEN ((allcum#94305710 = NA) OR (allcum#94305710 = null)) THEN null ELSE cast(allcum#94305710 as float) END AS allcum#94305869, CASE WHEN ((drawdown_gross#94305711 = NA) OR (drawdown_gross#94305711 = null)) THEN null ELSE cast(drawdown_gross#94305711 as float) END AS drawdown_gross#94305870, CASE WHEN ((drawdown_length_gross#94305712 = NA) OR (drawdown_length_gross#94305712 = null)) THEN null ELSE cast(drawdown_length_gross#94305712 as int) END AS drawdown_length_gross#94305872, CASE WHEN ((allretnet#94305713 = NA) OR (allretnet#94305713 = null)) THEN null ELSE cast(allretnet#94305713 as float) END AS allretnet#94305875, CASE WHEN ((allcumnet#94305714 = NA) OR (allcumnet#94305714 = null)) THEN null ELSE cast(allcumnet#94305714 as float) END AS allcumnet#94305877, CASE WHEN ((drawdown_net#94305715 = NA) OR (drawdown_net#94305715 = null)) THEN null ELSE cast(drawdown_net#94305715 as float) END AS drawdown_net#94305879, CASE WHEN ((drawdown_length_net#94305716 = NA) OR (drawdown_length_net#94305716 = null)) THEN null ELSE cast(drawdown_length_net#94305716 as int) END AS drawdown_length_net#94305881, CASE WHEN ((numcos#94305717 = NA) OR (numcos#94305717 = null)) THEN null ELSE cast(numcos#94305717 as float) END AS numcos#94305884, CASE WHEN ((largecum#94305718 = NA) OR (largecum#94305718 = null)) THEN null ELSE cast(largecum#94305718 as float) END AS largecum#94305886, CASE WHEN ((largecumnet#94305719 = NA) OR (largecumnet#94305719 = null)) THEN null ELSE cast(largecumnet#94305719 as float) END AS largecumnet#94305888, CASE WHEN ((largeret#94305720 = NA) OR (largeret#94305720 = null)) THEN null ELSE cast(largeret#94305720 as float) END AS largeret#94305890, CASE WHEN ((largeretnet#94305721 = NA) OR (largeretnet#94305721 = null)) THEN null ELSE cast(largeretnet#94305721 as float) END AS largeretnet#94305892, CASE WHEN ((midcum#94305722 = NA) OR (midcum#94305722 = null)) THEN null ELSE cast(midcum#94305722 as float) END AS midcum#94305894, CASE WHEN ((midcumnet#94305723 = NA) OR (midcumnet#94305723 = null)) THEN null ELSE cast(midcumnet#94305723 as float) END AS midcumnet#94305896, CASE WHEN ((midret#94305724 = NA) OR (midret#94305724 = null)) THEN null ELSE cast(midret#94305724 as float) END AS midret#94305898, CASE WHEN ((midretnet#94305725 = NA) OR (midretnet#94305725 = null)) THEN null ELSE cast(midretnet#94305725 as float) END AS midretnet#94305899, CASE WHEN ((smallcum#94305726 = NA) OR (smallcum#94305726 = null)) THEN null ELSE cast(smallcum#94305726 as float) END AS smallcum#94305902, CASE WHEN ((smallcumnet#94305727 = NA) OR (smallcumnet#94305727 = null)) THEN null ELSE cast(smallcumnet#94305727 as float) END AS smallcumnet#94305905, CASE WHEN ((smallret#94305728 = NA) OR (smallret#94305728 = null)) THEN null ELSE cast(smallret#94305728 as float) END AS smallret#94305907, CASE WHEN ((smallretnet#94305729 = NA) OR (smallretnet#94305729 = null)) THEN null ELSE cast(smallretnet#94305729 as float) END AS smallretnet#94305909]
Input [22]: [date#94305708, allret#94305709, allcum#94305710, drawdown_gross#94305711, drawdown_length_gross#94305712, allretnet#94305713, allcumnet#94305714, drawdown_net#94305715, drawdown_length_net#94305716, numcos#94305717, largecum#94305718, largecumnet#94305719, largeret#94305720, largeretnet#94305721, midcum#94305722, midcumnet#94305723, midret#94305724, midretnet#94305725, smallcum#94305726, smallcumnet#94305727, smallret#94305728, smallretnet#94305729]
(7) Exchange
Input [3]: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
Arguments: rangepartitioning(date#94305864 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7529657]
(8) Sort [codegen id : 1]
Input [3]: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
Arguments: [date#94305864 ASC NULLS FIRST], true, 0
(9) CollectLimit
Input [3]: [date#94305864, drawdown_gross#94305870, drawdown_net#94305879]
Arguments: 1000000