== Physical Plan ==
CollectLimit (9)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Sort (8)
+- Exchange (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [3]: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
Arguments: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
(2) InMemoryRelation
Arguments: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Sort [date#94153730 ASC NULLS FIRST], true, 0
+- Exchange rangepartitioning(date#94153730 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7517330]
+- InMemoryTableScan [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
+- InMemoryRelation [date#94153730, allret#94153731, allcum#94153946, drawdown_gross#94153953, drawdown_length_gross#94154064, allretnet#94154175, allcumnet#94154191, drawdown_net#94154192, drawdown_length_net#94154193, numcos#94154194, largecum#94154195, largecumnet#94154196, largeret#94154417, largeretnet#94154418, midcum#94154419, midcumnet#94154530, midret#94154641, midretnet#94154652, smallcum#94154653, smallcumnet#94154654, smallret#94154655, smallretnet#94154706], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN (date#94150544 = null) THEN null ELSE cast(date#94150544 as date) END AS date#94150771, CASE WHEN ((allret#94150545 = NA) OR (allret#94150545 = null)) THEN null ELSE cast(allret#94150545 as float) END AS allret#94150797, CASE WHEN ((allcum#94150546 = NA) OR (allcum#94150546 = null)) THEN null ELSE cast(allcum#94150546 as float) END AS allcum#94150805, CASE WHEN ((drawdown_gross#94150547 = NA) OR (drawdown_gross#94150547 = null)) THEN null ELSE cast(drawdown_gross#94150547 as float) END AS drawdown_gross#94150809, CASE WHEN ((drawdown_length_gross#94150548 = NA) OR (drawdown_length_gross#94150548 = null)) THEN null ELSE cast(drawdown_length_gross#94150548 as int) END AS drawdown_length_gross#94150837, CASE WHEN ((allretnet#94150549 = NA) OR (allretnet#94150549 = null)) THEN null ELSE cast(allretnet#94150549 as float) END AS allretnet#94150840, CASE WHEN ((allcumnet#94150550 = NA) OR (allcumnet#94150550 = null)) THEN null ELSE cast(allcumnet#94150550 as float) END AS allcumnet#94150846, CASE WHEN ((drawdown_net#94150551 = NA) OR (drawdown_net#94150551 = null)) THEN null ELSE cast(drawdown_net#94150551 as float) END AS drawdown_net#94150856, CASE WHEN ((drawdown_length_net#94150552 = NA) OR (drawdown_length_net#94150552 = null)) THEN null ELSE cast(drawdown_length_net#94150552 as int) END AS drawdown_length_net#94150863, CASE WHEN ((numcos#94150553 = NA) OR (numcos#94150553 = null)) THEN null ELSE cast(numcos#94150553 as float) END AS numcos#94150868, CASE WHEN ((largecum#94150554 = NA) OR (largecum#94150554 = null)) THEN null ELSE cast(largecum#94150554 as float) END AS largecum#94150871, CASE WHEN ((largecumnet#94150555 = NA) OR (largecumnet#94150555 = null)) THEN null ELSE cast(largecumnet#94150555 as float) END AS largecumnet#94150873, CASE WHEN ((largeret#94150556 = NA) OR (largeret#94150556 = null)) THEN null ELSE cast(largeret#94150556 as float) END AS largeret#94150875, CASE WHEN ((largeretnet#94150557 = NA) OR (largeretnet#94150557 = null)) THEN null ELSE cast(largeretnet#94150557 as float) END AS largeretnet#94150877, CASE WHEN ((midcum#94150558 = NA) OR (midcum#94150558 = null)) THEN null ELSE cast(midcum#94150558 as float) END AS midcum#94150879, CASE WHEN ((midcumnet#94150559 = NA) OR (midcumnet#94150559 = null)) THEN null ELSE cast(midcumnet#94150559 as float) END AS midcumnet#94150881, CASE WHEN ((midret#94150560 = NA) OR (midret#94150560 = null)) THEN null ELSE cast(midret#94150560 as float) END AS midret#94150884, CASE WHEN ((midretnet#94150561 = NA) OR (midretnet#94150561 = null)) THEN null ELSE cast(midretnet#94150561 as float) END AS midretnet#94150886, CASE WHEN ((smallcum#94150562 = NA) OR (smallcum#94150562 = null)) THEN null ELSE cast(smallcum#94150562 as float) END AS smallcum#94150914, CASE WHEN ((smallcumnet#94150563 = NA) OR (smallcumnet#94150563 = null)) THEN null ELSE cast(smallcumnet#94150563 as float) END AS smallcumnet#94150915, CASE WHEN ((smallret#94150564 = NA) OR (smallret#94150564 = null)) THEN null ELSE cast(smallret#94150564 as float) END AS smallret#94150916, CASE WHEN ((smallretnet#94150565 = NA) OR (smallretnet#94150565 = null)) THEN null ELSE cast(smallretnet#94150565 as float) END AS smallretnet#94150917]
+- FileScan csv [date#94150544,allret#94150545,allcum#94150546,drawdown_gross#94150547,drawdown_length_gross#94150548,allretnet#94150549,allcumnet#94150550,drawdown_net#94150551,drawdown_length_net#94150552,numcos#94150553,largecum#94150554,largecumnet#94150555,largeret#94150556,largeretnet#94150557,midcum#94150558,midcumnet#94150559,midret#94150560,midretnet#94150561,smallcum#94150562,smallcumnet#94150563,smallret#94150564,smallretnet#94150565] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/value/REP..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None), [date#94153730 ASC NULLS FIRST]
(3) InMemoryTableScan
Output [3]: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
Arguments: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
(4) InMemoryRelation
Arguments: [date#94153730, allret#94153731, allcum#94153946, drawdown_gross#94153953, drawdown_length_gross#94154064, allretnet#94154175, allcumnet#94154191, drawdown_net#94154192, drawdown_length_net#94154193, numcos#94154194, largecum#94154195, largecumnet#94154196, largeret#94154417, largeretnet#94154418, midcum#94154419, midcumnet#94154530, midret#94154641, midretnet#94154652, smallcum#94154653, smallcumnet#94154654, smallret#94154655, smallretnet#94154706], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94150544 = null) THEN null ELSE cast(date#94150544 as date) END AS date#94150771, CASE WHEN ((allret#94150545 = NA) OR (allret#94150545 = null)) THEN null ELSE cast(allret#94150545 as float) END AS allret#94150797, CASE WHEN ((allcum#94150546 = NA) OR (allcum#94150546 = null)) THEN null ELSE cast(allcum#94150546 as float) END AS allcum#94150805, CASE WHEN ((drawdown_gross#94150547 = NA) OR (drawdown_gross#94150547 = null)) THEN null ELSE cast(drawdown_gross#94150547 as float) END AS drawdown_gross#94150809, CASE WHEN ((drawdown_length_gross#94150548 = NA) OR (drawdown_length_gross#94150548 = null)) THEN null ELSE cast(drawdown_length_gross#94150548 as int) END AS drawdown_length_gross#94150837, CASE WHEN ((allretnet#94150549 = NA) OR (allretnet#94150549 = null)) THEN null ELSE cast(allretnet#94150549 as float) END AS allretnet#94150840, CASE WHEN ((allcumnet#94150550 = NA) OR (allcumnet#94150550 = null)) THEN null ELSE cast(allcumnet#94150550 as float) END AS allcumnet#94150846, CASE WHEN ((drawdown_net#94150551 = NA) OR (drawdown_net#94150551 = null)) THEN null ELSE cast(drawdown_net#94150551 as float) END AS drawdown_net#94150856, CASE WHEN ((drawdown_length_net#94150552 = NA) OR (drawdown_length_net#94150552 = null)) THEN null ELSE cast(drawdown_length_net#94150552 as int) END AS drawdown_length_net#94150863, CASE WHEN ((numcos#94150553 = NA) OR (numcos#94150553 = null)) THEN null ELSE cast(numcos#94150553 as float) END AS numcos#94150868, CASE WHEN ((largecum#94150554 = NA) OR (largecum#94150554 = null)) THEN null ELSE cast(largecum#94150554 as float) END AS largecum#94150871, CASE WHEN ((largecumnet#94150555 = NA) OR (largecumnet#94150555 = null)) THEN null ELSE cast(largecumnet#94150555 as float) END AS largecumnet#94150873, CASE WHEN ((largeret#94150556 = NA) OR (largeret#94150556 = null)) THEN null ELSE cast(largeret#94150556 as float) END AS largeret#94150875, CASE WHEN ((largeretnet#94150557 = NA) OR (largeretnet#94150557 = null)) THEN null ELSE cast(largeretnet#94150557 as float) END AS largeretnet#94150877, CASE WHEN ((midcum#94150558 = NA) OR (midcum#94150558 = null)) THEN null ELSE cast(midcum#94150558 as float) END AS midcum#94150879, CASE WHEN ((midcumnet#94150559 = NA) OR (midcumnet#94150559 = null)) THEN null ELSE cast(midcumnet#94150559 as float) END AS midcumnet#94150881, CASE WHEN ((midret#94150560 = NA) OR (midret#94150560 = null)) THEN null ELSE cast(midret#94150560 as float) END AS midret#94150884, CASE WHEN ((midretnet#94150561 = NA) OR (midretnet#94150561 = null)) THEN null ELSE cast(midretnet#94150561 as float) END AS midretnet#94150886, CASE WHEN ((smallcum#94150562 = NA) OR (smallcum#94150562 = null)) THEN null ELSE cast(smallcum#94150562 as float) END AS smallcum#94150914, CASE WHEN ((smallcumnet#94150563 = NA) OR (smallcumnet#94150563 = null)) THEN null ELSE cast(smallcumnet#94150563 as float) END AS smallcumnet#94150915, CASE WHEN ((smallret#94150564 = NA) OR (smallret#94150564 = null)) THEN null ELSE cast(smallret#94150564 as float) END AS smallret#94150916, CASE WHEN ((smallretnet#94150565 = NA) OR (smallretnet#94150565 = null)) THEN null ELSE cast(smallretnet#94150565 as float) END AS smallretnet#94150917]
+- FileScan csv [date#94150544,allret#94150545,allcum#94150546,drawdown_gross#94150547,drawdown_length_gross#94150548,allretnet#94150549,allcumnet#94150550,drawdown_net#94150551,drawdown_length_net#94150552,numcos#94150553,largecum#94150554,largecumnet#94150555,largeret#94150556,largeretnet#94150557,midcum#94150558,midcumnet#94150559,midret#94150560,midretnet#94150561,smallcum#94150562,smallcumnet#94150563,smallret#94150564,smallretnet#94150565] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/value/REP..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None)
(5) Scan csv
Output [22]: [date#94150544, allret#94150545, allcum#94150546, drawdown_gross#94150547, drawdown_length_gross#94150548, allretnet#94150549, allcumnet#94150550, drawdown_net#94150551, drawdown_length_net#94150552, numcos#94150553, largecum#94150554, largecumnet#94150555, largeret#94150556, largeretnet#94150557, midcum#94150558, midcumnet#94150559, midret#94150560, midretnet#94150561, smallcum#94150562, smallcumnet#94150563, smallret#94150564, smallretnet#94150565]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/value/REPORT_cumret.csv]
ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string>
(6) Project [codegen id : 1]
Output [22]: [CASE WHEN (date#94150544 = null) THEN null ELSE cast(date#94150544 as date) END AS date#94150771, CASE WHEN ((allret#94150545 = NA) OR (allret#94150545 = null)) THEN null ELSE cast(allret#94150545 as float) END AS allret#94150797, CASE WHEN ((allcum#94150546 = NA) OR (allcum#94150546 = null)) THEN null ELSE cast(allcum#94150546 as float) END AS allcum#94150805, CASE WHEN ((drawdown_gross#94150547 = NA) OR (drawdown_gross#94150547 = null)) THEN null ELSE cast(drawdown_gross#94150547 as float) END AS drawdown_gross#94150809, CASE WHEN ((drawdown_length_gross#94150548 = NA) OR (drawdown_length_gross#94150548 = null)) THEN null ELSE cast(drawdown_length_gross#94150548 as int) END AS drawdown_length_gross#94150837, CASE WHEN ((allretnet#94150549 = NA) OR (allretnet#94150549 = null)) THEN null ELSE cast(allretnet#94150549 as float) END AS allretnet#94150840, CASE WHEN ((allcumnet#94150550 = NA) OR (allcumnet#94150550 = null)) THEN null ELSE cast(allcumnet#94150550 as float) END AS allcumnet#94150846, CASE WHEN ((drawdown_net#94150551 = NA) OR (drawdown_net#94150551 = null)) THEN null ELSE cast(drawdown_net#94150551 as float) END AS drawdown_net#94150856, CASE WHEN ((drawdown_length_net#94150552 = NA) OR (drawdown_length_net#94150552 = null)) THEN null ELSE cast(drawdown_length_net#94150552 as int) END AS drawdown_length_net#94150863, CASE WHEN ((numcos#94150553 = NA) OR (numcos#94150553 = null)) THEN null ELSE cast(numcos#94150553 as float) END AS numcos#94150868, CASE WHEN ((largecum#94150554 = NA) OR (largecum#94150554 = null)) THEN null ELSE cast(largecum#94150554 as float) END AS largecum#94150871, CASE WHEN ((largecumnet#94150555 = NA) OR (largecumnet#94150555 = null)) THEN null ELSE cast(largecumnet#94150555 as float) END AS largecumnet#94150873, CASE WHEN ((largeret#94150556 = NA) OR (largeret#94150556 = null)) THEN null ELSE cast(largeret#94150556 as float) END AS largeret#94150875, CASE WHEN ((largeretnet#94150557 = NA) OR (largeretnet#94150557 = null)) THEN null ELSE cast(largeretnet#94150557 as float) END AS largeretnet#94150877, CASE WHEN ((midcum#94150558 = NA) OR (midcum#94150558 = null)) THEN null ELSE cast(midcum#94150558 as float) END AS midcum#94150879, CASE WHEN ((midcumnet#94150559 = NA) OR (midcumnet#94150559 = null)) THEN null ELSE cast(midcumnet#94150559 as float) END AS midcumnet#94150881, CASE WHEN ((midret#94150560 = NA) OR (midret#94150560 = null)) THEN null ELSE cast(midret#94150560 as float) END AS midret#94150884, CASE WHEN ((midretnet#94150561 = NA) OR (midretnet#94150561 = null)) THEN null ELSE cast(midretnet#94150561 as float) END AS midretnet#94150886, CASE WHEN ((smallcum#94150562 = NA) OR (smallcum#94150562 = null)) THEN null ELSE cast(smallcum#94150562 as float) END AS smallcum#94150914, CASE WHEN ((smallcumnet#94150563 = NA) OR (smallcumnet#94150563 = null)) THEN null ELSE cast(smallcumnet#94150563 as float) END AS smallcumnet#94150915, CASE WHEN ((smallret#94150564 = NA) OR (smallret#94150564 = null)) THEN null ELSE cast(smallret#94150564 as float) END AS smallret#94150916, CASE WHEN ((smallretnet#94150565 = NA) OR (smallretnet#94150565 = null)) THEN null ELSE cast(smallretnet#94150565 as float) END AS smallretnet#94150917]
Input [22]: [date#94150544, allret#94150545, allcum#94150546, drawdown_gross#94150547, drawdown_length_gross#94150548, allretnet#94150549, allcumnet#94150550, drawdown_net#94150551, drawdown_length_net#94150552, numcos#94150553, largecum#94150554, largecumnet#94150555, largeret#94150556, largeretnet#94150557, midcum#94150558, midcumnet#94150559, midret#94150560, midretnet#94150561, smallcum#94150562, smallcumnet#94150563, smallret#94150564, smallretnet#94150565]
(7) Exchange
Input [3]: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
Arguments: rangepartitioning(date#94153730 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7517330]
(8) Sort [codegen id : 1]
Input [3]: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
Arguments: [date#94153730 ASC NULLS FIRST], true, 0
(9) CollectLimit
Input [3]: [date#94153730, drawdown_gross#94153953, drawdown_net#94154192]
Arguments: 1000000