== Physical Plan ==
CollectLimit (10)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Sort (9)
+- Exchange (8)
+- * Project (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [3]: [date#94120626, daily_gross#94120960, daily_net#94120961]
Arguments: [date#94120626, daily_gross#94120960, daily_net#94120961]
(2) InMemoryRelation
Arguments: [date#94120626, daily_gross#94120960, daily_net#94120961], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94120626 ASC NULLS FIRST], true, 0
+- Exchange rangepartitioning(date#94120626 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7514838]
+- *(1) Project [date#94120626, allret#94120627 AS daily_gross#94120960, allretnet#94120631 AS daily_net#94120961]
+- InMemoryTableScan [allret#94120627, allretnet#94120631, date#94120626]
+- InMemoryRelation [date#94120626, allret#94120627, allcum#94120628, drawdown_gross#94120629, drawdown_length_gross#94120630, allretnet#94120631, allcumnet#94120632, drawdown_net#94120633, drawdown_length_net#94120634, numcos#94120635, largecum#94120636, largecumnet#94120637, largeret#94120639, largeretnet#94120640, midcum#94120641, midcumnet#94120642, midret#94120644, midretnet#94120645, smallcum#94120646, smallcumnet#94120669, smallret#94120692, smallretnet#94120693], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN (date#94119419 = null) THEN null ELSE cast(date#94119419 as date) END AS date#94119517, CASE WHEN ((allret#94119420 = NA) OR (allret#94119420 = null)) THEN null ELSE cast(allret#94119420 as float) END AS allret#94119518, CASE WHEN ((allcum#94119421 = NA) OR (allcum#94119421 = null)) THEN null ELSE cast(allcum#94119421 as float) END AS allcum#94119519, CASE WHEN ((drawdown_gross#94119422 = NA) OR (drawdown_gross#94119422 = null)) THEN null ELSE cast(drawdown_gross#94119422 as float) END AS drawdown_gross#94119520, CASE WHEN ((drawdown_length_gross#94119423 = NA) OR (drawdown_length_gross#94119423 = null)) THEN null ELSE cast(drawdown_length_gross#94119423 as int) END AS drawdown_length_gross#94119521, CASE WHEN ((allretnet#94119424 = NA) OR (allretnet#94119424 = null)) THEN null ELSE cast(allretnet#94119424 as float) END AS allretnet#94119522, CASE WHEN ((allcumnet#94119425 = NA) OR (allcumnet#94119425 = null)) THEN null ELSE cast(allcumnet#94119425 as float) END AS allcumnet#94119523, CASE WHEN ((drawdown_net#94119426 = NA) OR (drawdown_net#94119426 = null)) THEN null ELSE cast(drawdown_net#94119426 as float) END AS drawdown_net#94119524, CASE WHEN ((drawdown_length_net#94119427 = NA) OR (drawdown_length_net#94119427 = null)) THEN null ELSE cast(drawdown_length_net#94119427 as int) END AS drawdown_length_net#94119525, CASE WHEN ((numcos#94119428 = NA) OR (numcos#94119428 = null)) THEN null ELSE cast(numcos#94119428 as float) END AS numcos#94119527, CASE WHEN ((largecum#94119429 = NA) OR (largecum#94119429 = null)) THEN null ELSE cast(largecum#94119429 as float) END AS largecum#94119529, CASE WHEN ((largecumnet#94119430 = NA) OR (largecumnet#94119430 = null)) THEN null ELSE cast(largecumnet#94119430 as float) END AS largecumnet#94119555, CASE WHEN ((largeret#94119431 = NA) OR (largeret#94119431 = null)) THEN null ELSE cast(largeret#94119431 as float) END AS largeret#94119556, CASE WHEN ((largeretnet#94119432 = NA) OR (largeretnet#94119432 = null)) THEN null ELSE cast(largeretnet#94119432 as float) END AS largeretnet#94119557, CASE WHEN ((midcum#94119433 = NA) OR (midcum#94119433 = null)) THEN null ELSE cast(midcum#94119433 as float) END AS midcum#94119558, CASE WHEN ((midcumnet#94119434 = NA) OR (midcumnet#94119434 = null)) THEN null ELSE cast(midcumnet#94119434 as float) END AS midcumnet#94119559, CASE WHEN ((midret#94119435 = NA) OR (midret#94119435 = null)) THEN null ELSE cast(midret#94119435 as float) END AS midret#94119565, CASE WHEN ((midretnet#94119436 = NA) OR (midretnet#94119436 = null)) THEN null ELSE cast(midretnet#94119436 as float) END AS midretnet#94119570, CASE WHEN ((smallcum#94119437 = NA) OR (smallcum#94119437 = null)) THEN null ELSE cast(smallcum#94119437 as float) END AS smallcum#94119572, CASE WHEN ((smallcumnet#94119438 = NA) OR (smallcumnet#94119438 = null)) THEN null ELSE cast(smallcumnet#94119438 as float) END AS smallcumnet#94119578, CASE WHEN ((smallret#94119439 = NA) OR (smallret#94119439 = null)) THEN null ELSE cast(smallret#94119439 as float) END AS smallret#94119579, CASE WHEN ((smallretnet#94119440 = NA) OR (smallretnet#94119440 = null)) THEN null ELSE cast(smallretnet#94119440 as float) END AS smallretnet#94119585]
+- FileScan csv [date#94119419,allret#94119420,allcum#94119421,drawdown_gross#94119422,drawdown_length_gross#94119423,allretnet#94119424,allcumnet#94119425,drawdown_net#94119426,drawdown_length_net#94119427,numcos#94119428,largecum#94119429,largecumnet#94119430,largeret#94119431,largeretnet#94119432,midcum#94119433,midcumnet#94119434,midret#94119435,midretnet#94119436,smallcum#94119437,smallcumnet#94119438,smallret#94119439,smallretnet#94119440] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/transcripts/transcript..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None), [date#94120626 ASC NULLS FIRST]
(3) InMemoryTableScan
Output [3]: [allret#94120627, allretnet#94120631, date#94120626]
Arguments: [allret#94120627, allretnet#94120631, date#94120626]
(4) InMemoryRelation
Arguments: [date#94120626, allret#94120627, allcum#94120628, drawdown_gross#94120629, drawdown_length_gross#94120630, allretnet#94120631, allcumnet#94120632, drawdown_net#94120633, drawdown_length_net#94120634, numcos#94120635, largecum#94120636, largecumnet#94120637, largeret#94120639, largeretnet#94120640, midcum#94120641, midcumnet#94120642, midret#94120644, midretnet#94120645, smallcum#94120646, smallcumnet#94120669, smallret#94120692, smallretnet#94120693], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94119419 = null) THEN null ELSE cast(date#94119419 as date) END AS date#94119517, CASE WHEN ((allret#94119420 = NA) OR (allret#94119420 = null)) THEN null ELSE cast(allret#94119420 as float) END AS allret#94119518, CASE WHEN ((allcum#94119421 = NA) OR (allcum#94119421 = null)) THEN null ELSE cast(allcum#94119421 as float) END AS allcum#94119519, CASE WHEN ((drawdown_gross#94119422 = NA) OR (drawdown_gross#94119422 = null)) THEN null ELSE cast(drawdown_gross#94119422 as float) END AS drawdown_gross#94119520, CASE WHEN ((drawdown_length_gross#94119423 = NA) OR (drawdown_length_gross#94119423 = null)) THEN null ELSE cast(drawdown_length_gross#94119423 as int) END AS drawdown_length_gross#94119521, CASE WHEN ((allretnet#94119424 = NA) OR (allretnet#94119424 = null)) THEN null ELSE cast(allretnet#94119424 as float) END AS allretnet#94119522, CASE WHEN ((allcumnet#94119425 = NA) OR (allcumnet#94119425 = null)) THEN null ELSE cast(allcumnet#94119425 as float) END AS allcumnet#94119523, CASE WHEN ((drawdown_net#94119426 = NA) OR (drawdown_net#94119426 = null)) THEN null ELSE cast(drawdown_net#94119426 as float) END AS drawdown_net#94119524, CASE WHEN ((drawdown_length_net#94119427 = NA) OR (drawdown_length_net#94119427 = null)) THEN null ELSE cast(drawdown_length_net#94119427 as int) END AS drawdown_length_net#94119525, CASE WHEN ((numcos#94119428 = NA) OR (numcos#94119428 = null)) THEN null ELSE cast(numcos#94119428 as float) END AS numcos#94119527, CASE WHEN ((largecum#94119429 = NA) OR (largecum#94119429 = null)) THEN null ELSE cast(largecum#94119429 as float) END AS largecum#94119529, CASE WHEN ((largecumnet#94119430 = NA) OR (largecumnet#94119430 = null)) THEN null ELSE cast(largecumnet#94119430 as float) END AS largecumnet#94119555, CASE WHEN ((largeret#94119431 = NA) OR (largeret#94119431 = null)) THEN null ELSE cast(largeret#94119431 as float) END AS largeret#94119556, CASE WHEN ((largeretnet#94119432 = NA) OR (largeretnet#94119432 = null)) THEN null ELSE cast(largeretnet#94119432 as float) END AS largeretnet#94119557, CASE WHEN ((midcum#94119433 = NA) OR (midcum#94119433 = null)) THEN null ELSE cast(midcum#94119433 as float) END AS midcum#94119558, CASE WHEN ((midcumnet#94119434 = NA) OR (midcumnet#94119434 = null)) THEN null ELSE cast(midcumnet#94119434 as float) END AS midcumnet#94119559, CASE WHEN ((midret#94119435 = NA) OR (midret#94119435 = null)) THEN null ELSE cast(midret#94119435 as float) END AS midret#94119565, CASE WHEN ((midretnet#94119436 = NA) OR (midretnet#94119436 = null)) THEN null ELSE cast(midretnet#94119436 as float) END AS midretnet#94119570, CASE WHEN ((smallcum#94119437 = NA) OR (smallcum#94119437 = null)) THEN null ELSE cast(smallcum#94119437 as float) END AS smallcum#94119572, CASE WHEN ((smallcumnet#94119438 = NA) OR (smallcumnet#94119438 = null)) THEN null ELSE cast(smallcumnet#94119438 as float) END AS smallcumnet#94119578, CASE WHEN ((smallret#94119439 = NA) OR (smallret#94119439 = null)) THEN null ELSE cast(smallret#94119439 as float) END AS smallret#94119579, CASE WHEN ((smallretnet#94119440 = NA) OR (smallretnet#94119440 = null)) THEN null ELSE cast(smallretnet#94119440 as float) END AS smallretnet#94119585]
+- FileScan csv [date#94119419,allret#94119420,allcum#94119421,drawdown_gross#94119422,drawdown_length_gross#94119423,allretnet#94119424,allcumnet#94119425,drawdown_net#94119426,drawdown_length_net#94119427,numcos#94119428,largecum#94119429,largecumnet#94119430,largeret#94119431,largeretnet#94119432,midcum#94119433,midcumnet#94119434,midret#94119435,midretnet#94119436,smallcum#94119437,smallcumnet#94119438,smallret#94119439,smallretnet#94119440] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/transcripts/transcript..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string...
,None)
(5) Scan csv
Output [22]: [date#94119419, allret#94119420, allcum#94119421, drawdown_gross#94119422, drawdown_length_gross#94119423, allretnet#94119424, allcumnet#94119425, drawdown_net#94119426, drawdown_length_net#94119427, numcos#94119428, largecum#94119429, largecumnet#94119430, largeret#94119431, largeretnet#94119432, midcum#94119433, midcumnet#94119434, midret#94119435, midretnet#94119436, smallcum#94119437, smallcumnet#94119438, smallret#94119439, smallretnet#94119440]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/transcripts/transcript_model_residualized/REPORT_cumret.csv]
ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string>
(6) Project [codegen id : 1]
Output [22]: [CASE WHEN (date#94119419 = null) THEN null ELSE cast(date#94119419 as date) END AS date#94119517, CASE WHEN ((allret#94119420 = NA) OR (allret#94119420 = null)) THEN null ELSE cast(allret#94119420 as float) END AS allret#94119518, CASE WHEN ((allcum#94119421 = NA) OR (allcum#94119421 = null)) THEN null ELSE cast(allcum#94119421 as float) END AS allcum#94119519, CASE WHEN ((drawdown_gross#94119422 = NA) OR (drawdown_gross#94119422 = null)) THEN null ELSE cast(drawdown_gross#94119422 as float) END AS drawdown_gross#94119520, CASE WHEN ((drawdown_length_gross#94119423 = NA) OR (drawdown_length_gross#94119423 = null)) THEN null ELSE cast(drawdown_length_gross#94119423 as int) END AS drawdown_length_gross#94119521, CASE WHEN ((allretnet#94119424 = NA) OR (allretnet#94119424 = null)) THEN null ELSE cast(allretnet#94119424 as float) END AS allretnet#94119522, CASE WHEN ((allcumnet#94119425 = NA) OR (allcumnet#94119425 = null)) THEN null ELSE cast(allcumnet#94119425 as float) END AS allcumnet#94119523, CASE WHEN ((drawdown_net#94119426 = NA) OR (drawdown_net#94119426 = null)) THEN null ELSE cast(drawdown_net#94119426 as float) END AS drawdown_net#94119524, CASE WHEN ((drawdown_length_net#94119427 = NA) OR (drawdown_length_net#94119427 = null)) THEN null ELSE cast(drawdown_length_net#94119427 as int) END AS drawdown_length_net#94119525, CASE WHEN ((numcos#94119428 = NA) OR (numcos#94119428 = null)) THEN null ELSE cast(numcos#94119428 as float) END AS numcos#94119527, CASE WHEN ((largecum#94119429 = NA) OR (largecum#94119429 = null)) THEN null ELSE cast(largecum#94119429 as float) END AS largecum#94119529, CASE WHEN ((largecumnet#94119430 = NA) OR (largecumnet#94119430 = null)) THEN null ELSE cast(largecumnet#94119430 as float) END AS largecumnet#94119555, CASE WHEN ((largeret#94119431 = NA) OR (largeret#94119431 = null)) THEN null ELSE cast(largeret#94119431 as float) END AS largeret#94119556, CASE WHEN ((largeretnet#94119432 = NA) OR (largeretnet#94119432 = null)) THEN null ELSE cast(largeretnet#94119432 as float) END AS largeretnet#94119557, CASE WHEN ((midcum#94119433 = NA) OR (midcum#94119433 = null)) THEN null ELSE cast(midcum#94119433 as float) END AS midcum#94119558, CASE WHEN ((midcumnet#94119434 = NA) OR (midcumnet#94119434 = null)) THEN null ELSE cast(midcumnet#94119434 as float) END AS midcumnet#94119559, CASE WHEN ((midret#94119435 = NA) OR (midret#94119435 = null)) THEN null ELSE cast(midret#94119435 as float) END AS midret#94119565, CASE WHEN ((midretnet#94119436 = NA) OR (midretnet#94119436 = null)) THEN null ELSE cast(midretnet#94119436 as float) END AS midretnet#94119570, CASE WHEN ((smallcum#94119437 = NA) OR (smallcum#94119437 = null)) THEN null ELSE cast(smallcum#94119437 as float) END AS smallcum#94119572, CASE WHEN ((smallcumnet#94119438 = NA) OR (smallcumnet#94119438 = null)) THEN null ELSE cast(smallcumnet#94119438 as float) END AS smallcumnet#94119578, CASE WHEN ((smallret#94119439 = NA) OR (smallret#94119439 = null)) THEN null ELSE cast(smallret#94119439 as float) END AS smallret#94119579, CASE WHEN ((smallretnet#94119440 = NA) OR (smallretnet#94119440 = null)) THEN null ELSE cast(smallretnet#94119440 as float) END AS smallretnet#94119585]
Input [22]: [date#94119419, allret#94119420, allcum#94119421, drawdown_gross#94119422, drawdown_length_gross#94119423, allretnet#94119424, allcumnet#94119425, drawdown_net#94119426, drawdown_length_net#94119427, numcos#94119428, largecum#94119429, largecumnet#94119430, largeret#94119431, largeretnet#94119432, midcum#94119433, midcumnet#94119434, midret#94119435, midretnet#94119436, smallcum#94119437, smallcumnet#94119438, smallret#94119439, smallretnet#94119440]
(7) Project [codegen id : 1]
Output [3]: [date#94120626, allret#94120627 AS daily_gross#94120960, allretnet#94120631 AS daily_net#94120961]
Input [3]: [allret#94120627, allretnet#94120631, date#94120626]
(8) Exchange
Input [3]: [date#94120626, daily_gross#94120960, daily_net#94120961]
Arguments: rangepartitioning(date#94120626 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7514838]
(9) Sort [codegen id : 2]
Input [3]: [date#94120626, daily_gross#94120960, daily_net#94120961]
Arguments: [date#94120626 ASC NULLS FIRST], true, 0
(10) CollectLimit
Input [3]: [date#94120626, daily_gross#94120960, daily_net#94120961]
Arguments: 1000000