== Physical Plan == CollectLimit (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Sort (9) +- Exchange (8) +- * Project (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [3]: [date#94377294, cum_gross#94377473, cum_net#94377474] Arguments: [date#94377294, cum_gross#94377473, cum_net#94377474] (2) InMemoryRelation Arguments: [date#94377294, cum_gross#94377473, cum_net#94377474], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94377294 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94377294 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535304] +- *(1) Project [date#94377294, allcum#94377301 AS cum_gross#94377473, allcumnet#94377317 AS cum_net#94377474] +- InMemoryTableScan [allcum#94377301, allcumnet#94377317, date#94377294] +- InMemoryRelation [date#94377294, allret#94377300, allcum#94377301, drawdown_gross#94377307, drawdown_length_gross#94377308, allretnet#94377314, allcumnet#94377317, drawdown_net#94377323, drawdown_length_net#94377324, numcos#94377325, largecum#94377326, largecumnet#94377327, largeret#94377328, largeretnet#94377329, midcum#94377355, midcumnet#94377356, midret#94377361, midretnet#94377362, smallcum#94377363, smallcumnet#94377364, smallret#94377365, smallretnet#94377366], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN (date#94377170 = null) THEN null ELSE cast(date#94377170 as date) END AS date#94377294, CASE WHEN ((allret#94377171 = NA) OR (allret#94377171 = null)) THEN null ELSE cast(allret#94377171 as float) END AS allret#94377300, CASE WHEN ((allcum#94377172 = NA) OR (allcum#94377172 = null)) THEN null ELSE cast(allcum#94377172 as float) END AS allcum#94377301, CASE WHEN ((drawdown_gross#94377173 = NA) OR (drawdown_gross#94377173 = null)) THEN null ELSE cast(drawdown_gross#94377173 as float) END AS drawdown_gross#94377307, CASE WHEN ((drawdown_length_gross#94377174 = NA) OR (drawdown_length_gross#94377174 = null)) THEN null ELSE cast(drawdown_length_gross#94377174 as int) END AS drawdown_length_gross#94377308, CASE WHEN ((allretnet#94377175 = NA) OR (allretnet#94377175 = null)) THEN null ELSE cast(allretnet#94377175 as float) END AS allretnet#94377314, CASE WHEN ((allcumnet#94377176 = NA) OR (allcumnet#94377176 = null)) THEN null ELSE cast(allcumnet#94377176 as float) END AS allcumnet#94377317, CASE WHEN ((drawdown_net#94377177 = NA) OR (drawdown_net#94377177 = null)) THEN null ELSE cast(drawdown_net#94377177 as float) END AS drawdown_net#94377323, CASE WHEN ((drawdown_length_net#94377178 = NA) OR (drawdown_length_net#94377178 = null)) THEN null ELSE cast(drawdown_length_net#94377178 as int) END AS drawdown_length_net#94377324, CASE WHEN ((numcos#94377179 = NA) OR (numcos#94377179 = null)) THEN null ELSE cast(numcos#94377179 as float) END AS numcos#94377325, CASE WHEN ((largecum#94377180 = NA) OR (largecum#94377180 = null)) THEN null ELSE cast(largecum#94377180 as float) END AS largecum#94377326, CASE WHEN ((largecumnet#94377181 = NA) OR (largecumnet#94377181 = null)) THEN null ELSE cast(largecumnet#94377181 as float) END AS largecumnet#94377327, CASE WHEN ((largeret#94377182 = NA) OR (largeret#94377182 = null)) THEN null ELSE cast(largeret#94377182 as float) END AS largeret#94377328, CASE WHEN ((largeretnet#94377183 = NA) OR (largeretnet#94377183 = null)) THEN null ELSE cast(largeretnet#94377183 as float) END AS largeretnet#94377329, CASE WHEN ((midcum#94377184 = NA) OR (midcum#94377184 = null)) THEN null ELSE cast(midcum#94377184 as float) END AS midcum#94377355, CASE WHEN ((midcumnet#94377185 = NA) OR (midcumnet#94377185 = null)) THEN null ELSE cast(midcumnet#94377185 as float) END AS midcumnet#94377356, CASE WHEN ((midret#94377186 = NA) OR (midret#94377186 = null)) THEN null ELSE cast(midret#94377186 as float) END AS midret#94377361, CASE WHEN ((midretnet#94377187 = NA) OR (midretnet#94377187 = null)) THEN null ELSE cast(midretnet#94377187 as float) END AS midretnet#94377362, CASE WHEN ((smallcum#94377188 = NA) OR (smallcum#94377188 = null)) THEN null ELSE cast(smallcum#94377188 as float) END AS smallcum#94377363, CASE WHEN ((smallcumnet#94377189 = NA) OR (smallcumnet#94377189 = null)) THEN null ELSE cast(smallcumnet#94377189 as float) END AS smallcumnet#94377364, CASE WHEN ((smallret#94377190 = NA) OR (smallret#94377190 = null)) THEN null ELSE cast(smallret#94377190 as float) END AS smallret#94377365, CASE WHEN ((smallretnet#94377191 = NA) OR (smallretnet#94377191 = null)) THEN null ELSE cast(smallretnet#94377191 as float) END AS smallretnet#94377366] +- FileScan csv [date#94377170,allret#94377171,allcum#94377172,drawdown_gross#94377173,drawdown_length_gross#94377174,allretnet#94377175,allcumnet#94377176,drawdown_net#94377177,drawdown_length_net#94377178,numcos#94377179,largecum#94377180,largecumnet#94377181,largeret#94377182,largeretnet#94377183,midcum#94377184,midcumnet#94377185,midret#94377186,midretnet#94377187,smallcum#94377188,smallcumnet#94377189,smallret#94377190,smallretnet#94377191] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None), [date#94377294 ASC NULLS FIRST] (3) InMemoryTableScan Output [3]: [allcum#94377301, allcumnet#94377317, date#94377294] Arguments: [allcum#94377301, allcumnet#94377317, date#94377294] (4) InMemoryRelation Arguments: [date#94377294, allret#94377300, allcum#94377301, drawdown_gross#94377307, drawdown_length_gross#94377308, allretnet#94377314, allcumnet#94377317, drawdown_net#94377323, drawdown_length_net#94377324, numcos#94377325, largecum#94377326, largecumnet#94377327, largeret#94377328, largeretnet#94377329, midcum#94377355, midcumnet#94377356, midret#94377361, midretnet#94377362, smallcum#94377363, smallcumnet#94377364, smallret#94377365, smallretnet#94377366], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94377170 = null) THEN null ELSE cast(date#94377170 as date) END AS date#94377294, CASE WHEN ((allret#94377171 = NA) OR (allret#94377171 = null)) THEN null ELSE cast(allret#94377171 as float) END AS allret#94377300, CASE WHEN ((allcum#94377172 = NA) OR (allcum#94377172 = null)) THEN null ELSE cast(allcum#94377172 as float) END AS allcum#94377301, CASE WHEN ((drawdown_gross#94377173 = NA) OR (drawdown_gross#94377173 = null)) THEN null ELSE cast(drawdown_gross#94377173 as float) END AS drawdown_gross#94377307, CASE WHEN ((drawdown_length_gross#94377174 = NA) OR (drawdown_length_gross#94377174 = null)) THEN null ELSE cast(drawdown_length_gross#94377174 as int) END AS drawdown_length_gross#94377308, CASE WHEN ((allretnet#94377175 = NA) OR (allretnet#94377175 = null)) THEN null ELSE cast(allretnet#94377175 as float) END AS allretnet#94377314, CASE WHEN ((allcumnet#94377176 = NA) OR (allcumnet#94377176 = null)) THEN null ELSE cast(allcumnet#94377176 as float) END AS allcumnet#94377317, CASE WHEN ((drawdown_net#94377177 = NA) OR (drawdown_net#94377177 = null)) THEN null ELSE cast(drawdown_net#94377177 as float) END AS drawdown_net#94377323, CASE WHEN ((drawdown_length_net#94377178 = NA) OR (drawdown_length_net#94377178 = null)) THEN null ELSE cast(drawdown_length_net#94377178 as int) END AS drawdown_length_net#94377324, CASE WHEN ((numcos#94377179 = NA) OR (numcos#94377179 = null)) THEN null ELSE cast(numcos#94377179 as float) END AS numcos#94377325, CASE WHEN ((largecum#94377180 = NA) OR (largecum#94377180 = null)) THEN null ELSE cast(largecum#94377180 as float) END AS largecum#94377326, CASE WHEN ((largecumnet#94377181 = NA) OR (largecumnet#94377181 = null)) THEN null ELSE cast(largecumnet#94377181 as float) END AS largecumnet#94377327, CASE WHEN ((largeret#94377182 = NA) OR (largeret#94377182 = null)) THEN null ELSE cast(largeret#94377182 as float) END AS largeret#94377328, CASE WHEN ((largeretnet#94377183 = NA) OR (largeretnet#94377183 = null)) THEN null ELSE cast(largeretnet#94377183 as float) END AS largeretnet#94377329, CASE WHEN ((midcum#94377184 = NA) OR (midcum#94377184 = null)) THEN null ELSE cast(midcum#94377184 as float) END AS midcum#94377355, CASE WHEN ((midcumnet#94377185 = NA) OR (midcumnet#94377185 = null)) THEN null ELSE cast(midcumnet#94377185 as float) END AS midcumnet#94377356, CASE WHEN ((midret#94377186 = NA) OR (midret#94377186 = null)) THEN null ELSE cast(midret#94377186 as float) END AS midret#94377361, CASE WHEN ((midretnet#94377187 = NA) OR (midretnet#94377187 = null)) THEN null ELSE cast(midretnet#94377187 as float) END AS midretnet#94377362, CASE WHEN ((smallcum#94377188 = NA) OR (smallcum#94377188 = null)) THEN null ELSE cast(smallcum#94377188 as float) END AS smallcum#94377363, CASE WHEN ((smallcumnet#94377189 = NA) OR (smallcumnet#94377189 = null)) THEN null ELSE cast(smallcumnet#94377189 as float) END AS smallcumnet#94377364, CASE WHEN ((smallret#94377190 = NA) OR (smallret#94377190 = null)) THEN null ELSE cast(smallret#94377190 as float) END AS smallret#94377365, CASE WHEN ((smallretnet#94377191 = NA) OR (smallretnet#94377191 = null)) THEN null ELSE cast(smallretnet#94377191 as float) END AS smallretnet#94377366] +- FileScan csv [date#94377170,allret#94377171,allcum#94377172,drawdown_gross#94377173,drawdown_length_gross#94377174,allretnet#94377175,allcumnet#94377176,drawdown_net#94377177,drawdown_length_net#94377178,numcos#94377179,largecum#94377180,largecumnet#94377181,largeret#94377182,largeretnet#94377183,midcum#94377184,midcumnet#94377185,midret#94377186,midretnet#94377187,smallcum#94377188,smallcumnet#94377189,smallret#94377190,smallretnet#94377191] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string... ,None) (5) Scan csv Output [22]: [date#94377170, allret#94377171, allcum#94377172, drawdown_gross#94377173, drawdown_length_gross#94377174, allretnet#94377175, allcumnet#94377176, drawdown_net#94377177, drawdown_length_net#94377178, numcos#94377179, largecum#94377180, largecumnet#94377181, largeret#94377182, largeretnet#94377183, midcum#94377184, midcumnet#94377185, midret#94377186, midretnet#94377187, smallcum#94377188, smallcumnet#94377189, smallret#94377190, smallretnet#94377191] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/REPORT_cumret.csv] ReadSchema: struct<date:string,allret:string,allcum:string,drawdown_gross:string,drawdown_length_gross:string,allretnet:string,allcumnet:string,drawdown_net:string,drawdown_length_net:string,numcos:string,largecum:string,largecumnet:string,largeret:string,largeretnet:string,midcum:string,midcumnet:string,midret:string,midretnet:string,smallcum:string,smallcumnet:string,smallret:string,smallretnet:string> (6) Project [codegen id : 1] Output [22]: [CASE WHEN (date#94377170 = null) THEN null ELSE cast(date#94377170 as date) END AS date#94377294, CASE WHEN ((allret#94377171 = NA) OR (allret#94377171 = null)) THEN null ELSE cast(allret#94377171 as float) END AS allret#94377300, CASE WHEN ((allcum#94377172 = NA) OR (allcum#94377172 = null)) THEN null ELSE cast(allcum#94377172 as float) END AS allcum#94377301, CASE WHEN ((drawdown_gross#94377173 = NA) OR (drawdown_gross#94377173 = null)) THEN null ELSE cast(drawdown_gross#94377173 as float) END AS drawdown_gross#94377307, CASE WHEN ((drawdown_length_gross#94377174 = NA) OR (drawdown_length_gross#94377174 = null)) THEN null ELSE cast(drawdown_length_gross#94377174 as int) END AS drawdown_length_gross#94377308, CASE WHEN ((allretnet#94377175 = NA) OR (allretnet#94377175 = null)) THEN null ELSE cast(allretnet#94377175 as float) END AS allretnet#94377314, CASE WHEN ((allcumnet#94377176 = NA) OR (allcumnet#94377176 = null)) THEN null ELSE cast(allcumnet#94377176 as float) END AS allcumnet#94377317, CASE WHEN ((drawdown_net#94377177 = NA) OR (drawdown_net#94377177 = null)) THEN null ELSE cast(drawdown_net#94377177 as float) END AS drawdown_net#94377323, CASE WHEN ((drawdown_length_net#94377178 = NA) OR (drawdown_length_net#94377178 = null)) THEN null ELSE cast(drawdown_length_net#94377178 as int) END AS drawdown_length_net#94377324, CASE WHEN ((numcos#94377179 = NA) OR (numcos#94377179 = null)) THEN null ELSE cast(numcos#94377179 as float) END AS numcos#94377325, CASE WHEN ((largecum#94377180 = NA) OR (largecum#94377180 = null)) THEN null ELSE cast(largecum#94377180 as float) END AS largecum#94377326, CASE WHEN ((largecumnet#94377181 = NA) OR (largecumnet#94377181 = null)) THEN null ELSE cast(largecumnet#94377181 as float) END AS largecumnet#94377327, CASE WHEN ((largeret#94377182 = NA) OR (largeret#94377182 = null)) THEN null ELSE cast(largeret#94377182 as float) END AS largeret#94377328, CASE WHEN ((largeretnet#94377183 = NA) OR (largeretnet#94377183 = null)) THEN null ELSE cast(largeretnet#94377183 as float) END AS largeretnet#94377329, CASE WHEN ((midcum#94377184 = NA) OR (midcum#94377184 = null)) THEN null ELSE cast(midcum#94377184 as float) END AS midcum#94377355, CASE WHEN ((midcumnet#94377185 = NA) OR (midcumnet#94377185 = null)) THEN null ELSE cast(midcumnet#94377185 as float) END AS midcumnet#94377356, CASE WHEN ((midret#94377186 = NA) OR (midret#94377186 = null)) THEN null ELSE cast(midret#94377186 as float) END AS midret#94377361, CASE WHEN ((midretnet#94377187 = NA) OR (midretnet#94377187 = null)) THEN null ELSE cast(midretnet#94377187 as float) END AS midretnet#94377362, CASE WHEN ((smallcum#94377188 = NA) OR (smallcum#94377188 = null)) THEN null ELSE cast(smallcum#94377188 as float) END AS smallcum#94377363, CASE WHEN ((smallcumnet#94377189 = NA) OR (smallcumnet#94377189 = null)) THEN null ELSE cast(smallcumnet#94377189 as float) END AS smallcumnet#94377364, CASE WHEN ((smallret#94377190 = NA) OR (smallret#94377190 = null)) THEN null ELSE cast(smallret#94377190 as float) END AS smallret#94377365, CASE WHEN ((smallretnet#94377191 = NA) OR (smallretnet#94377191 = null)) THEN null ELSE cast(smallretnet#94377191 as float) END AS smallretnet#94377366] Input [22]: [date#94377170, allret#94377171, allcum#94377172, drawdown_gross#94377173, drawdown_length_gross#94377174, allretnet#94377175, allcumnet#94377176, drawdown_net#94377177, drawdown_length_net#94377178, numcos#94377179, largecum#94377180, largecumnet#94377181, largeret#94377182, largeretnet#94377183, midcum#94377184, midcumnet#94377185, midret#94377186, midretnet#94377187, smallcum#94377188, smallcumnet#94377189, smallret#94377190, smallretnet#94377191] (7) Project [codegen id : 1] Output [3]: [date#94377294, allcum#94377301 AS cum_gross#94377473, allcumnet#94377317 AS cum_net#94377474] Input [3]: [allcum#94377301, allcumnet#94377317, date#94377294] (8) Exchange Input [3]: [date#94377294, cum_gross#94377473, cum_net#94377474] Arguments: rangepartitioning(date#94377294 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535304] (9) Sort [codegen id : 2] Input [3]: [date#94377294, cum_gross#94377473, cum_net#94377474] Arguments: [date#94377294 ASC NULLS FIRST], true, 0 (10) CollectLimit Input [3]: [date#94377294, cum_gross#94377473, cum_net#94377474] Arguments: 1000000