== Physical Plan == CollectLimit (11) +- * ColumnarToRow (10) +- InMemoryTableScan (1) +- InMemoryRelation (2) +- * Project (9) +- * Filter (8) +- * ColumnarToRow (7) +- InMemoryTableScan (3) +- InMemoryRelation (4) +- * Project (6) +- Scan csv (5) (1) InMemoryTableScan Output [7]: [yield#94028387, volatility#94028384, momentum#94028386, size#94028376, value#94028378, growth#94028379, leverage#94028382] Arguments: [yield#94028387, volatility#94028384, momentum#94028386, size#94028376, value#94028378, growth#94028379, leverage#94028382] (2) InMemoryRelation Arguments: [yield#94028387, volatility#94028384, momentum#94028386, size#94028376, value#94028378, growth#94028379, leverage#94028382], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [yield#94028387, volatility#94028384, momentum#94028386, size#94028376, value#94028378, growth#94028379, leverage#94028382] +- *(1) Filter (isnotnull(cap#94028370) AND (cap#94028370 = 0.0)) +- *(1) ColumnarToRow +- InMemoryTableScan [cap#94028370, growth#94028379, leverage#94028382, momentum#94028386, size#94028376, value#94028378, volatility#94028384, yield#94028387], [isnotnull(cap#94028370), (cap#94028370 = 0.0)] +- InMemoryRelation [overall#94028368, cap#94028370, retIC#94028372, resretIC#94028374, size#94028376, value#94028378, growth#94028379, leverage#94028382, volatility#94028384, momentum#94028386, yield#94028387, numcos#94028389, numdates#94028391, annual_bmret#94028393, annual_ret#94028395, std_ret#94028397, Sharpe_ret#94028399, PctPos_ret#94028402, TR_ret#94028403, IR_ret#94028405, annual_resret#94028407, std_resret#94028409, Sharpe_resret#94028410, PctPos_resret#94028411, ... 9 more fields], StorageLevel(disk, memory, deserialized, 1 replicas) +- *(1) Project [CASE WHEN ((overall#94028181 = NA) OR (overall#94028181 = null)) THEN null ELSE cast(overall#94028181 as int) END AS overall#94028368, CASE WHEN ((cap#94028182 = NA) OR (cap#94028182 = null)) THEN null ELSE cast(cap#94028182 as float) END AS cap#94028370, CASE WHEN ((retIC#94028183 = NA) OR (retIC#94028183 = null)) THEN null ELSE cast(retIC#94028183 as float) END AS retIC#94028372, CASE WHEN ((resretIC#94028184 = NA) OR (resretIC#94028184 = null)) THEN null ELSE cast(resretIC#94028184 as float) END AS resretIC#94028374, CASE WHEN ((size#94028185 = NA) OR (size#94028185 = null)) THEN null ELSE cast(size#94028185 as float) END AS size#94028376, CASE WHEN ((value#94028186 = NA) OR (value#94028186 = null)) THEN null ELSE cast(value#94028186 as float) END AS value#94028378, CASE WHEN ((growth#94028187 = NA) OR (growth#94028187 = null)) THEN null ELSE cast(growth#94028187 as float) END AS growth#94028379, CASE WHEN ((leverage#94028188 = NA) OR (leverage#94028188 = null)) THEN null ELSE cast(leverage#94028188 as float) END AS leverage#94028382, CASE WHEN ((volatility#94028189 = NA) OR (volatility#94028189 = null)) THEN null ELSE cast(volatility#94028189 as float) END AS volatility#94028384, CASE WHEN ((momentum#94028190 = NA) OR (momentum#94028190 = null)) THEN null ELSE cast(momentum#94028190 as float) END AS momentum#94028386, CASE WHEN ((yield#94028191 = NA) OR (yield#94028191 = null)) THEN null ELSE cast(yield#94028191 as float) END AS yield#94028387, CASE WHEN ((numcos#94028192 = NA) OR (numcos#94028192 = null)) THEN null ELSE cast(numcos#94028192 as float) END AS numcos#94028389, CASE WHEN ((numdates#94028193 = NA) OR (numdates#94028193 = null)) THEN null ELSE cast(numdates#94028193 as float) END AS numdates#94028391, CASE WHEN ((annual_bmret#94028194 = NA) OR (annual_bmret#94028194 = null)) THEN null ELSE cast(annual_bmret#94028194 as float) END AS annual_bmret#94028393, CASE WHEN ((annual_ret#94028195 = NA) OR (annual_ret#94028195 = null)) THEN null ELSE cast(annual_ret#94028195 as float) END AS annual_ret#94028395, CASE WHEN ((std_ret#94028196 = NA) OR (std_ret#94028196 = null)) THEN null ELSE cast(std_ret#94028196 as float) END AS std_ret#94028397, CASE WHEN ((Sharpe_ret#94028197 = NA) OR (Sharpe_ret#94028197 = null)) THEN null ELSE cast(Sharpe_ret#94028197 as float) END AS Sharpe_ret#94028399, CASE WHEN ((PctPos_ret#94028198 = NA) OR (PctPos_ret#94028198 = null)) THEN null ELSE cast(PctPos_ret#94028198 as float) END AS PctPos_ret#94028402, CASE WHEN ((TR_ret#94028199 = NA) OR (TR_ret#94028199 = null)) THEN null ELSE cast(TR_ret#94028199 as float) END AS TR_ret#94028403, CASE WHEN ((IR_ret#94028200 = NA) OR (IR_ret#94028200 = null)) THEN null ELSE cast(IR_ret#94028200 as float) END AS IR_ret#94028405, CASE WHEN ((annual_resret#94028201 = NA) OR (annual_resret#94028201 = null)) THEN null ELSE cast(annual_resret#94028201 as float) END AS annual_resret#94028407, CASE WHEN ((std_resret#94028202 = NA) OR (std_resret#94028202 = null)) THEN null ELSE cast(std_resret#94028202 as float) END AS std_resret#94028409, CASE WHEN ((Sharpe_resret#94028203 = NA) OR (Sharpe_resret#94028203 = null)) THEN null ELSE cast(Sharpe_resret#94028203 as float) END AS Sharpe_resret#94028410, CASE WHEN ((PctPos_resret#94028204 = NA) OR (PctPos_resret#94028204 = null)) THEN null ELSE cast(PctPos_resret#94028204 as float) END AS PctPos_resret#94028411, ... 9 more fields] +- FileScan csv [overall#94028181,cap#94028182,retIC#94028183,resretIC#94028184,size#94028185,value#94028186,growth#94028187,leverage#94028188,volatility#94028189,momentum#94028190,yield#94028191,numcos#94028192,numdates#94028193,annual_bmret#94028194,annual_ret#94028195,std_ret#94028196,Sharpe_ret#94028197,PctPos_ret#94028198,TR_ret#94028199,IR_ret#94028200,annual_resret#94028201,std_resret#94028202,Sharpe_resret#94028203,PctPos_resret#94028204,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/st..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str... ,None) (3) InMemoryTableScan Output [8]: [cap#94028370, growth#94028379, leverage#94028382, momentum#94028386, size#94028376, value#94028378, volatility#94028384, yield#94028387] Arguments: [cap#94028370, growth#94028379, leverage#94028382, momentum#94028386, size#94028376, value#94028378, volatility#94028384, yield#94028387], [isnotnull(cap#94028370), (cap#94028370 = 0.0)] (4) InMemoryRelation Arguments: [overall#94028368, cap#94028370, retIC#94028372, resretIC#94028374, size#94028376, value#94028378, growth#94028379, leverage#94028382, volatility#94028384, momentum#94028386, yield#94028387, numcos#94028389, numdates#94028391, annual_bmret#94028393, annual_ret#94028395, std_ret#94028397, Sharpe_ret#94028399, PctPos_ret#94028402, TR_ret#94028403, IR_ret#94028405, annual_resret#94028407, std_resret#94028409, Sharpe_resret#94028410, PctPos_resret#94028411, ... 9 more fields], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((overall#94028181 = NA) OR (overall#94028181 = null)) THEN null ELSE cast(overall#94028181 as int) END AS overall#94028368, CASE WHEN ((cap#94028182 = NA) OR (cap#94028182 = null)) THEN null ELSE cast(cap#94028182 as float) END AS cap#94028370, CASE WHEN ((retIC#94028183 = NA) OR (retIC#94028183 = null)) THEN null ELSE cast(retIC#94028183 as float) END AS retIC#94028372, CASE WHEN ((resretIC#94028184 = NA) OR (resretIC#94028184 = null)) THEN null ELSE cast(resretIC#94028184 as float) END AS resretIC#94028374, CASE WHEN ((size#94028185 = NA) OR (size#94028185 = null)) THEN null ELSE cast(size#94028185 as float) END AS size#94028376, CASE WHEN ((value#94028186 = NA) OR (value#94028186 = null)) THEN null ELSE cast(value#94028186 as float) END AS value#94028378, CASE WHEN ((growth#94028187 = NA) OR (growth#94028187 = null)) THEN null ELSE cast(growth#94028187 as float) END AS growth#94028379, CASE WHEN ((leverage#94028188 = NA) OR (leverage#94028188 = null)) THEN null ELSE cast(leverage#94028188 as float) END AS leverage#94028382, CASE WHEN ((volatility#94028189 = NA) OR (volatility#94028189 = null)) THEN null ELSE cast(volatility#94028189 as float) END AS volatility#94028384, CASE WHEN ((momentum#94028190 = NA) OR (momentum#94028190 = null)) THEN null ELSE cast(momentum#94028190 as float) END AS momentum#94028386, CASE WHEN ((yield#94028191 = NA) OR (yield#94028191 = null)) THEN null ELSE cast(yield#94028191 as float) END AS yield#94028387, CASE WHEN ((numcos#94028192 = NA) OR (numcos#94028192 = null)) THEN null ELSE cast(numcos#94028192 as float) END AS numcos#94028389, CASE WHEN ((numdates#94028193 = NA) OR (numdates#94028193 = null)) THEN null ELSE cast(numdates#94028193 as float) END AS numdates#94028391, CASE WHEN ((annual_bmret#94028194 = NA) OR (annual_bmret#94028194 = null)) THEN null ELSE cast(annual_bmret#94028194 as float) END AS annual_bmret#94028393, CASE WHEN ((annual_ret#94028195 = NA) OR (annual_ret#94028195 = null)) THEN null ELSE cast(annual_ret#94028195 as float) END AS annual_ret#94028395, CASE WHEN ((std_ret#94028196 = NA) OR (std_ret#94028196 = null)) THEN null ELSE cast(std_ret#94028196 as float) END AS std_ret#94028397, CASE WHEN ((Sharpe_ret#94028197 = NA) OR (Sharpe_ret#94028197 = null)) THEN null ELSE cast(Sharpe_ret#94028197 as float) END AS Sharpe_ret#94028399, CASE WHEN ((PctPos_ret#94028198 = NA) OR (PctPos_ret#94028198 = null)) THEN null ELSE cast(PctPos_ret#94028198 as float) END AS PctPos_ret#94028402, CASE WHEN ((TR_ret#94028199 = NA) OR (TR_ret#94028199 = null)) THEN null ELSE cast(TR_ret#94028199 as float) END AS TR_ret#94028403, CASE WHEN ((IR_ret#94028200 = NA) OR (IR_ret#94028200 = null)) THEN null ELSE cast(IR_ret#94028200 as float) END AS IR_ret#94028405, CASE WHEN ((annual_resret#94028201 = NA) OR (annual_resret#94028201 = null)) THEN null ELSE cast(annual_resret#94028201 as float) END AS annual_resret#94028407, CASE WHEN ((std_resret#94028202 = NA) OR (std_resret#94028202 = null)) THEN null ELSE cast(std_resret#94028202 as float) END AS std_resret#94028409, CASE WHEN ((Sharpe_resret#94028203 = NA) OR (Sharpe_resret#94028203 = null)) THEN null ELSE cast(Sharpe_resret#94028203 as float) END AS Sharpe_resret#94028410, CASE WHEN ((PctPos_resret#94028204 = NA) OR (PctPos_resret#94028204 = null)) THEN null ELSE cast(PctPos_resret#94028204 as float) END AS PctPos_resret#94028411, ... 9 more fields] +- FileScan csv [overall#94028181,cap#94028182,retIC#94028183,resretIC#94028184,size#94028185,value#94028186,growth#94028187,leverage#94028188,volatility#94028189,momentum#94028190,yield#94028191,numcos#94028192,numdates#94028193,annual_bmret#94028194,annual_ret#94028195,std_ret#94028196,Sharpe_ret#94028197,PctPos_ret#94028198,TR_ret#94028199,IR_ret#94028200,annual_resret#94028201,std_resret#94028202,Sharpe_resret#94028203,PctPos_resret#94028204,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/st..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str... ,None) (5) Scan csv Output [33]: [overall#94028181, cap#94028182, retIC#94028183, resretIC#94028184, size#94028185, value#94028186, growth#94028187, leverage#94028188, volatility#94028189, momentum#94028190, yield#94028191, numcos#94028192, numdates#94028193, annual_bmret#94028194, annual_ret#94028195, std_ret#94028196, Sharpe_ret#94028197, PctPos_ret#94028198, TR_ret#94028199, IR_ret#94028200, annual_resret#94028201, std_resret#94028202, Sharpe_resret#94028203, PctPos_resret#94028204, TR_resret#94028205, IR_resret#94028206, annual_retnet#94028207, std_retnet#94028208, Sharpe_retnet#94028209, PctPos_retnet#94028210, TR_retnet#94028211, IR_retnet#94028212, turnover#94028213] Batched: false Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/stats_overall.csv] ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:string,leverage:string,volatility:string,momentum:string,yield:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string> (6) Project [codegen id : 1] Output [33]: [CASE WHEN ((overall#94028181 = NA) OR (overall#94028181 = null)) THEN null ELSE cast(overall#94028181 as int) END AS overall#94028368, CASE WHEN ((cap#94028182 = NA) OR (cap#94028182 = null)) THEN null ELSE cast(cap#94028182 as float) END AS cap#94028370, CASE WHEN ((retIC#94028183 = NA) OR (retIC#94028183 = null)) THEN null ELSE cast(retIC#94028183 as float) END AS retIC#94028372, CASE WHEN ((resretIC#94028184 = NA) OR (resretIC#94028184 = null)) THEN null ELSE cast(resretIC#94028184 as float) END AS resretIC#94028374, CASE WHEN ((size#94028185 = NA) OR (size#94028185 = null)) THEN null ELSE cast(size#94028185 as float) END AS size#94028376, CASE WHEN ((value#94028186 = NA) OR (value#94028186 = null)) THEN null ELSE cast(value#94028186 as float) END AS value#94028378, CASE WHEN ((growth#94028187 = NA) OR (growth#94028187 = null)) THEN null ELSE cast(growth#94028187 as float) END AS growth#94028379, CASE WHEN ((leverage#94028188 = NA) OR (leverage#94028188 = null)) THEN null ELSE cast(leverage#94028188 as float) END AS leverage#94028382, CASE WHEN ((volatility#94028189 = NA) OR (volatility#94028189 = null)) THEN null ELSE cast(volatility#94028189 as float) END AS volatility#94028384, CASE WHEN ((momentum#94028190 = NA) OR (momentum#94028190 = null)) THEN null ELSE cast(momentum#94028190 as float) END AS momentum#94028386, CASE WHEN ((yield#94028191 = NA) OR (yield#94028191 = null)) THEN null ELSE cast(yield#94028191 as float) END AS yield#94028387, CASE WHEN ((numcos#94028192 = NA) OR (numcos#94028192 = null)) THEN null ELSE cast(numcos#94028192 as float) END AS numcos#94028389, CASE WHEN ((numdates#94028193 = NA) OR (numdates#94028193 = null)) THEN null ELSE cast(numdates#94028193 as float) END AS numdates#94028391, CASE WHEN ((annual_bmret#94028194 = NA) OR (annual_bmret#94028194 = null)) THEN null ELSE cast(annual_bmret#94028194 as float) END AS annual_bmret#94028393, CASE WHEN ((annual_ret#94028195 = NA) OR (annual_ret#94028195 = null)) THEN null ELSE cast(annual_ret#94028195 as float) END AS annual_ret#94028395, CASE WHEN ((std_ret#94028196 = NA) OR (std_ret#94028196 = null)) THEN null ELSE cast(std_ret#94028196 as float) END AS std_ret#94028397, CASE WHEN ((Sharpe_ret#94028197 = NA) OR (Sharpe_ret#94028197 = null)) THEN null ELSE cast(Sharpe_ret#94028197 as float) END AS Sharpe_ret#94028399, CASE WHEN ((PctPos_ret#94028198 = NA) OR (PctPos_ret#94028198 = null)) THEN null ELSE cast(PctPos_ret#94028198 as float) END AS PctPos_ret#94028402, CASE WHEN ((TR_ret#94028199 = NA) OR (TR_ret#94028199 = null)) THEN null ELSE cast(TR_ret#94028199 as float) END AS TR_ret#94028403, CASE WHEN ((IR_ret#94028200 = NA) OR (IR_ret#94028200 = null)) THEN null ELSE cast(IR_ret#94028200 as float) END AS IR_ret#94028405, CASE WHEN ((annual_resret#94028201 = NA) OR (annual_resret#94028201 = null)) THEN null ELSE cast(annual_resret#94028201 as float) END AS annual_resret#94028407, CASE WHEN ((std_resret#94028202 = NA) OR (std_resret#94028202 = null)) THEN null ELSE cast(std_resret#94028202 as float) END AS std_resret#94028409, CASE WHEN ((Sharpe_resret#94028203 = NA) OR (Sharpe_resret#94028203 = null)) THEN null ELSE cast(Sharpe_resret#94028203 as float) END AS Sharpe_resret#94028410, CASE WHEN ((PctPos_resret#94028204 = NA) OR (PctPos_resret#94028204 = null)) THEN null ELSE cast(PctPos_resret#94028204 as float) END AS PctPos_resret#94028411, CASE WHEN ((TR_resret#94028205 = NA) OR (TR_resret#94028205 = null)) THEN null ELSE cast(TR_resret#94028205 as float) END AS TR_resret#94028412, CASE WHEN ((IR_resret#94028206 = NA) OR (IR_resret#94028206 = null)) THEN null ELSE cast(IR_resret#94028206 as float) END AS IR_resret#94028413, CASE WHEN ((annual_retnet#94028207 = NA) OR (annual_retnet#94028207 = null)) THEN null ELSE cast(annual_retnet#94028207 as float) END AS annual_retnet#94028414, CASE WHEN ((std_retnet#94028208 = NA) OR (std_retnet#94028208 = null)) THEN null ELSE cast(std_retnet#94028208 as float) END AS std_retnet#94028415, CASE WHEN ((Sharpe_retnet#94028209 = NA) OR (Sharpe_retnet#94028209 = null)) THEN null ELSE cast(Sharpe_retnet#94028209 as float) END AS Sharpe_retnet#94028416, CASE WHEN ((PctPos_retnet#94028210 = NA) OR (PctPos_retnet#94028210 = null)) THEN null ELSE cast(PctPos_retnet#94028210 as float) END AS PctPos_retnet#94028417, CASE WHEN ((TR_retnet#94028211 = NA) OR (TR_retnet#94028211 = null)) THEN null ELSE cast(TR_retnet#94028211 as float) END AS TR_retnet#94028418, CASE WHEN ((IR_retnet#94028212 = NA) OR (IR_retnet#94028212 = null)) THEN null ELSE cast(IR_retnet#94028212 as float) END AS IR_retnet#94028419, CASE WHEN ((turnover#94028213 = NA) OR (turnover#94028213 = null)) THEN null ELSE cast(turnover#94028213 as float) END AS turnover#94028420] Input [33]: [overall#94028181, cap#94028182, retIC#94028183, resretIC#94028184, size#94028185, value#94028186, growth#94028187, leverage#94028188, volatility#94028189, momentum#94028190, yield#94028191, numcos#94028192, numdates#94028193, annual_bmret#94028194, annual_ret#94028195, std_ret#94028196, Sharpe_ret#94028197, PctPos_ret#94028198, TR_ret#94028199, IR_ret#94028200, annual_resret#94028201, std_resret#94028202, Sharpe_resret#94028203, PctPos_resret#94028204, TR_resret#94028205, IR_resret#94028206, annual_retnet#94028207, std_retnet#94028208, Sharpe_retnet#94028209, PctPos_retnet#94028210, TR_retnet#94028211, IR_retnet#94028212, turnover#94028213] (7) ColumnarToRow [codegen id : 1] Input [8]: [cap#94028370, growth#94028379, leverage#94028382, momentum#94028386, size#94028376, value#94028378, volatility#94028384, yield#94028387] (8) Filter [codegen id : 1] Input [8]: [cap#94028370, growth#94028379, leverage#94028382, momentum#94028386, size#94028376, value#94028378, volatility#94028384, yield#94028387] Condition : (isnotnull(cap#94028370) AND (cap#94028370 = 0.0)) (9) Project [codegen id : 1] Output [7]: [yield#94028387, volatility#94028384, momentum#94028386, size#94028376, value#94028378, growth#94028379, leverage#94028382] Input [8]: [cap#94028370, growth#94028379, leverage#94028382, momentum#94028386, size#94028376, value#94028378, volatility#94028384, yield#94028387] (10) ColumnarToRow [codegen id : 1] Input [7]: [yield#94028387, volatility#94028384, momentum#94028386, size#94028376, value#94028378, growth#94028379, leverage#94028382] (11) CollectLimit Input [7]: [yield#94028387, volatility#94028384, momentum#94028386, size#94028376, value#94028378, growth#94028379, leverage#94028382] Arguments: 1000000