== Physical Plan ==
CollectLimit (11)
+- * ColumnarToRow (10)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (9)
+- * Filter (8)
+- * ColumnarToRow (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [7]: [yield#94057303, volatility#94057299, momentum#94057301, size#94057292, value#94057294, growth#94057296, leverage#94057298]
Arguments: [yield#94057303, volatility#94057299, momentum#94057301, size#94057292, value#94057294, growth#94057296, leverage#94057298]
(2) InMemoryRelation
Arguments: [yield#94057303, volatility#94057299, momentum#94057301, size#94057292, value#94057294, growth#94057296, leverage#94057298], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [yield#94057303, volatility#94057299, momentum#94057301, size#94057292, value#94057294, growth#94057296, leverage#94057298]
+- *(1) Filter (isnotnull(cap#94057287) AND (cap#94057287 = 0.0))
+- *(1) ColumnarToRow
+- InMemoryTableScan [cap#94057287, growth#94057296, leverage#94057298, momentum#94057301, size#94057292, value#94057294, volatility#94057299, yield#94057303], [isnotnull(cap#94057287), (cap#94057287 = 0.0)]
+- InMemoryRelation [overall#94057285, cap#94057287, retIC#94057289, resretIC#94057291, size#94057292, value#94057294, growth#94057296, leverage#94057298, volatility#94057299, momentum#94057301, yield#94057303, numcos#94057305, numdates#94057306, annual_bmret#94057308, annual_ret#94057310, std_ret#94057311, Sharpe_ret#94057313, PctPos_ret#94057315, TR_ret#94057317, IR_ret#94057319, annual_resret#94057320, std_resret#94057322, Sharpe_resret#94057324, PctPos_resret#94057325, ... 9 more fields], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN ((overall#94057145 = NA) OR (overall#94057145 = null)) THEN null ELSE cast(overall#94057145 as int) END AS overall#94057285, CASE WHEN ((cap#94057146 = NA) OR (cap#94057146 = null)) THEN null ELSE cast(cap#94057146 as float) END AS cap#94057287, CASE WHEN ((retIC#94057147 = NA) OR (retIC#94057147 = null)) THEN null ELSE cast(retIC#94057147 as float) END AS retIC#94057289, CASE WHEN ((resretIC#94057148 = NA) OR (resretIC#94057148 = null)) THEN null ELSE cast(resretIC#94057148 as float) END AS resretIC#94057291, CASE WHEN ((size#94057149 = NA) OR (size#94057149 = null)) THEN null ELSE cast(size#94057149 as float) END AS size#94057292, CASE WHEN ((value#94057150 = NA) OR (value#94057150 = null)) THEN null ELSE cast(value#94057150 as float) END AS value#94057294, CASE WHEN ((growth#94057151 = NA) OR (growth#94057151 = null)) THEN null ELSE cast(growth#94057151 as float) END AS growth#94057296, CASE WHEN ((leverage#94057152 = NA) OR (leverage#94057152 = null)) THEN null ELSE cast(leverage#94057152 as float) END AS leverage#94057298, CASE WHEN ((volatility#94057153 = NA) OR (volatility#94057153 = null)) THEN null ELSE cast(volatility#94057153 as float) END AS volatility#94057299, CASE WHEN ((momentum#94057154 = NA) OR (momentum#94057154 = null)) THEN null ELSE cast(momentum#94057154 as float) END AS momentum#94057301, CASE WHEN ((yield#94057155 = NA) OR (yield#94057155 = null)) THEN null ELSE cast(yield#94057155 as float) END AS yield#94057303, CASE WHEN ((numcos#94057156 = NA) OR (numcos#94057156 = null)) THEN null ELSE cast(numcos#94057156 as float) END AS numcos#94057305, CASE WHEN ((numdates#94057157 = NA) OR (numdates#94057157 = null)) THEN null ELSE cast(numdates#94057157 as float) END AS numdates#94057306, CASE WHEN ((annual_bmret#94057158 = NA) OR (annual_bmret#94057158 = null)) THEN null ELSE cast(annual_bmret#94057158 as float) END AS annual_bmret#94057308, CASE WHEN ((annual_ret#94057159 = NA) OR (annual_ret#94057159 = null)) THEN null ELSE cast(annual_ret#94057159 as float) END AS annual_ret#94057310, CASE WHEN ((std_ret#94057160 = NA) OR (std_ret#94057160 = null)) THEN null ELSE cast(std_ret#94057160 as float) END AS std_ret#94057311, CASE WHEN ((Sharpe_ret#94057161 = NA) OR (Sharpe_ret#94057161 = null)) THEN null ELSE cast(Sharpe_ret#94057161 as float) END AS Sharpe_ret#94057313, CASE WHEN ((PctPos_ret#94057162 = NA) OR (PctPos_ret#94057162 = null)) THEN null ELSE cast(PctPos_ret#94057162 as float) END AS PctPos_ret#94057315, CASE WHEN ((TR_ret#94057163 = NA) OR (TR_ret#94057163 = null)) THEN null ELSE cast(TR_ret#94057163 as float) END AS TR_ret#94057317, CASE WHEN ((IR_ret#94057164 = NA) OR (IR_ret#94057164 = null)) THEN null ELSE cast(IR_ret#94057164 as float) END AS IR_ret#94057319, CASE WHEN ((annual_resret#94057165 = NA) OR (annual_resret#94057165 = null)) THEN null ELSE cast(annual_resret#94057165 as float) END AS annual_resret#94057320, CASE WHEN ((std_resret#94057166 = NA) OR (std_resret#94057166 = null)) THEN null ELSE cast(std_resret#94057166 as float) END AS std_resret#94057322, CASE WHEN ((Sharpe_resret#94057167 = NA) OR (Sharpe_resret#94057167 = null)) THEN null ELSE cast(Sharpe_resret#94057167 as float) END AS Sharpe_resret#94057324, CASE WHEN ((PctPos_resret#94057168 = NA) OR (PctPos_resret#94057168 = null)) THEN null ELSE cast(PctPos_resret#94057168 as float) END AS PctPos_resret#94057325, ... 9 more fields]
+- FileScan csv [overall#94057145,cap#94057146,retIC#94057147,resretIC#94057148,size#94057149,value#94057150,growth#94057151,leverage#94057152,volatility#94057153,momentum#94057154,yield#94057155,numcos#94057156,numdates#94057157,annual_bmret#94057158,annual_ret#94057159,std_ret#94057160,Sharpe_ret#94057161,PctPos_ret#94057162,TR_ret#94057163,IR_ret#94057164,annual_resret#94057165,std_resret#94057166,Sharpe_resret#94057167,PctPos_resret#94057168,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(3) InMemoryTableScan
Output [8]: [cap#94057287, growth#94057296, leverage#94057298, momentum#94057301, size#94057292, value#94057294, volatility#94057299, yield#94057303]
Arguments: [cap#94057287, growth#94057296, leverage#94057298, momentum#94057301, size#94057292, value#94057294, volatility#94057299, yield#94057303], [isnotnull(cap#94057287), (cap#94057287 = 0.0)]
(4) InMemoryRelation
Arguments: [overall#94057285, cap#94057287, retIC#94057289, resretIC#94057291, size#94057292, value#94057294, growth#94057296, leverage#94057298, volatility#94057299, momentum#94057301, yield#94057303, numcos#94057305, numdates#94057306, annual_bmret#94057308, annual_ret#94057310, std_ret#94057311, Sharpe_ret#94057313, PctPos_ret#94057315, TR_ret#94057317, IR_ret#94057319, annual_resret#94057320, std_resret#94057322, Sharpe_resret#94057324, PctPos_resret#94057325, ... 9 more fields], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((overall#94057145 = NA) OR (overall#94057145 = null)) THEN null ELSE cast(overall#94057145 as int) END AS overall#94057285, CASE WHEN ((cap#94057146 = NA) OR (cap#94057146 = null)) THEN null ELSE cast(cap#94057146 as float) END AS cap#94057287, CASE WHEN ((retIC#94057147 = NA) OR (retIC#94057147 = null)) THEN null ELSE cast(retIC#94057147 as float) END AS retIC#94057289, CASE WHEN ((resretIC#94057148 = NA) OR (resretIC#94057148 = null)) THEN null ELSE cast(resretIC#94057148 as float) END AS resretIC#94057291, CASE WHEN ((size#94057149 = NA) OR (size#94057149 = null)) THEN null ELSE cast(size#94057149 as float) END AS size#94057292, CASE WHEN ((value#94057150 = NA) OR (value#94057150 = null)) THEN null ELSE cast(value#94057150 as float) END AS value#94057294, CASE WHEN ((growth#94057151 = NA) OR (growth#94057151 = null)) THEN null ELSE cast(growth#94057151 as float) END AS growth#94057296, CASE WHEN ((leverage#94057152 = NA) OR (leverage#94057152 = null)) THEN null ELSE cast(leverage#94057152 as float) END AS leverage#94057298, CASE WHEN ((volatility#94057153 = NA) OR (volatility#94057153 = null)) THEN null ELSE cast(volatility#94057153 as float) END AS volatility#94057299, CASE WHEN ((momentum#94057154 = NA) OR (momentum#94057154 = null)) THEN null ELSE cast(momentum#94057154 as float) END AS momentum#94057301, CASE WHEN ((yield#94057155 = NA) OR (yield#94057155 = null)) THEN null ELSE cast(yield#94057155 as float) END AS yield#94057303, CASE WHEN ((numcos#94057156 = NA) OR (numcos#94057156 = null)) THEN null ELSE cast(numcos#94057156 as float) END AS numcos#94057305, CASE WHEN ((numdates#94057157 = NA) OR (numdates#94057157 = null)) THEN null ELSE cast(numdates#94057157 as float) END AS numdates#94057306, CASE WHEN ((annual_bmret#94057158 = NA) OR (annual_bmret#94057158 = null)) THEN null ELSE cast(annual_bmret#94057158 as float) END AS annual_bmret#94057308, CASE WHEN ((annual_ret#94057159 = NA) OR (annual_ret#94057159 = null)) THEN null ELSE cast(annual_ret#94057159 as float) END AS annual_ret#94057310, CASE WHEN ((std_ret#94057160 = NA) OR (std_ret#94057160 = null)) THEN null ELSE cast(std_ret#94057160 as float) END AS std_ret#94057311, CASE WHEN ((Sharpe_ret#94057161 = NA) OR (Sharpe_ret#94057161 = null)) THEN null ELSE cast(Sharpe_ret#94057161 as float) END AS Sharpe_ret#94057313, CASE WHEN ((PctPos_ret#94057162 = NA) OR (PctPos_ret#94057162 = null)) THEN null ELSE cast(PctPos_ret#94057162 as float) END AS PctPos_ret#94057315, CASE WHEN ((TR_ret#94057163 = NA) OR (TR_ret#94057163 = null)) THEN null ELSE cast(TR_ret#94057163 as float) END AS TR_ret#94057317, CASE WHEN ((IR_ret#94057164 = NA) OR (IR_ret#94057164 = null)) THEN null ELSE cast(IR_ret#94057164 as float) END AS IR_ret#94057319, CASE WHEN ((annual_resret#94057165 = NA) OR (annual_resret#94057165 = null)) THEN null ELSE cast(annual_resret#94057165 as float) END AS annual_resret#94057320, CASE WHEN ((std_resret#94057166 = NA) OR (std_resret#94057166 = null)) THEN null ELSE cast(std_resret#94057166 as float) END AS std_resret#94057322, CASE WHEN ((Sharpe_resret#94057167 = NA) OR (Sharpe_resret#94057167 = null)) THEN null ELSE cast(Sharpe_resret#94057167 as float) END AS Sharpe_resret#94057324, CASE WHEN ((PctPos_resret#94057168 = NA) OR (PctPos_resret#94057168 = null)) THEN null ELSE cast(PctPos_resret#94057168 as float) END AS PctPos_resret#94057325, ... 9 more fields]
+- FileScan csv [overall#94057145,cap#94057146,retIC#94057147,resretIC#94057148,size#94057149,value#94057150,growth#94057151,leverage#94057152,volatility#94057153,momentum#94057154,yield#94057155,numcos#94057156,numdates#94057157,annual_bmret#94057158,annual_ret#94057159,std_ret#94057160,Sharpe_ret#94057161,PctPos_ret#94057162,TR_ret#94057163,IR_ret#94057164,annual_resret#94057165,std_resret#94057166,Sharpe_resret#94057167,PctPos_resret#94057168,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(5) Scan csv
Output [33]: [overall#94057145, cap#94057146, retIC#94057147, resretIC#94057148, size#94057149, value#94057150, growth#94057151, leverage#94057152, volatility#94057153, momentum#94057154, yield#94057155, numcos#94057156, numdates#94057157, annual_bmret#94057158, annual_ret#94057159, std_ret#94057160, Sharpe_ret#94057161, PctPos_ret#94057162, TR_ret#94057163, IR_ret#94057164, annual_resret#94057165, std_resret#94057166, Sharpe_resret#94057167, PctPos_resret#94057168, TR_resret#94057169, IR_resret#94057170, annual_retnet#94057171, std_retnet#94057172, Sharpe_retnet#94057173, PctPos_retnet#94057174, TR_retnet#94057175, IR_retnet#94057176, turnover#94057177]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/stats_overall.csv]
ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:string,leverage:string,volatility:string,momentum:string,yield:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string>
(6) Project [codegen id : 1]
Output [33]: [CASE WHEN ((overall#94057145 = NA) OR (overall#94057145 = null)) THEN null ELSE cast(overall#94057145 as int) END AS overall#94057285, CASE WHEN ((cap#94057146 = NA) OR (cap#94057146 = null)) THEN null ELSE cast(cap#94057146 as float) END AS cap#94057287, CASE WHEN ((retIC#94057147 = NA) OR (retIC#94057147 = null)) THEN null ELSE cast(retIC#94057147 as float) END AS retIC#94057289, CASE WHEN ((resretIC#94057148 = NA) OR (resretIC#94057148 = null)) THEN null ELSE cast(resretIC#94057148 as float) END AS resretIC#94057291, CASE WHEN ((size#94057149 = NA) OR (size#94057149 = null)) THEN null ELSE cast(size#94057149 as float) END AS size#94057292, CASE WHEN ((value#94057150 = NA) OR (value#94057150 = null)) THEN null ELSE cast(value#94057150 as float) END AS value#94057294, CASE WHEN ((growth#94057151 = NA) OR (growth#94057151 = null)) THEN null ELSE cast(growth#94057151 as float) END AS growth#94057296, CASE WHEN ((leverage#94057152 = NA) OR (leverage#94057152 = null)) THEN null ELSE cast(leverage#94057152 as float) END AS leverage#94057298, CASE WHEN ((volatility#94057153 = NA) OR (volatility#94057153 = null)) THEN null ELSE cast(volatility#94057153 as float) END AS volatility#94057299, CASE WHEN ((momentum#94057154 = NA) OR (momentum#94057154 = null)) THEN null ELSE cast(momentum#94057154 as float) END AS momentum#94057301, CASE WHEN ((yield#94057155 = NA) OR (yield#94057155 = null)) THEN null ELSE cast(yield#94057155 as float) END AS yield#94057303, CASE WHEN ((numcos#94057156 = NA) OR (numcos#94057156 = null)) THEN null ELSE cast(numcos#94057156 as float) END AS numcos#94057305, CASE WHEN ((numdates#94057157 = NA) OR (numdates#94057157 = null)) THEN null ELSE cast(numdates#94057157 as float) END AS numdates#94057306, CASE WHEN ((annual_bmret#94057158 = NA) OR (annual_bmret#94057158 = null)) THEN null ELSE cast(annual_bmret#94057158 as float) END AS annual_bmret#94057308, CASE WHEN ((annual_ret#94057159 = NA) OR (annual_ret#94057159 = null)) THEN null ELSE cast(annual_ret#94057159 as float) END AS annual_ret#94057310, CASE WHEN ((std_ret#94057160 = NA) OR (std_ret#94057160 = null)) THEN null ELSE cast(std_ret#94057160 as float) END AS std_ret#94057311, CASE WHEN ((Sharpe_ret#94057161 = NA) OR (Sharpe_ret#94057161 = null)) THEN null ELSE cast(Sharpe_ret#94057161 as float) END AS Sharpe_ret#94057313, CASE WHEN ((PctPos_ret#94057162 = NA) OR (PctPos_ret#94057162 = null)) THEN null ELSE cast(PctPos_ret#94057162 as float) END AS PctPos_ret#94057315, CASE WHEN ((TR_ret#94057163 = NA) OR (TR_ret#94057163 = null)) THEN null ELSE cast(TR_ret#94057163 as float) END AS TR_ret#94057317, CASE WHEN ((IR_ret#94057164 = NA) OR (IR_ret#94057164 = null)) THEN null ELSE cast(IR_ret#94057164 as float) END AS IR_ret#94057319, CASE WHEN ((annual_resret#94057165 = NA) OR (annual_resret#94057165 = null)) THEN null ELSE cast(annual_resret#94057165 as float) END AS annual_resret#94057320, CASE WHEN ((std_resret#94057166 = NA) OR (std_resret#94057166 = null)) THEN null ELSE cast(std_resret#94057166 as float) END AS std_resret#94057322, CASE WHEN ((Sharpe_resret#94057167 = NA) OR (Sharpe_resret#94057167 = null)) THEN null ELSE cast(Sharpe_resret#94057167 as float) END AS Sharpe_resret#94057324, CASE WHEN ((PctPos_resret#94057168 = NA) OR (PctPos_resret#94057168 = null)) THEN null ELSE cast(PctPos_resret#94057168 as float) END AS PctPos_resret#94057325, CASE WHEN ((TR_resret#94057169 = NA) OR (TR_resret#94057169 = null)) THEN null ELSE cast(TR_resret#94057169 as float) END AS TR_resret#94057326, CASE WHEN ((IR_resret#94057170 = NA) OR (IR_resret#94057170 = null)) THEN null ELSE cast(IR_resret#94057170 as float) END AS IR_resret#94057327, CASE WHEN ((annual_retnet#94057171 = NA) OR (annual_retnet#94057171 = null)) THEN null ELSE cast(annual_retnet#94057171 as float) END AS annual_retnet#94057328, CASE WHEN ((std_retnet#94057172 = NA) OR (std_retnet#94057172 = null)) THEN null ELSE cast(std_retnet#94057172 as float) END AS std_retnet#94057329, CASE WHEN ((Sharpe_retnet#94057173 = NA) OR (Sharpe_retnet#94057173 = null)) THEN null ELSE cast(Sharpe_retnet#94057173 as float) END AS Sharpe_retnet#94057330, CASE WHEN ((PctPos_retnet#94057174 = NA) OR (PctPos_retnet#94057174 = null)) THEN null ELSE cast(PctPos_retnet#94057174 as float) END AS PctPos_retnet#94057331, CASE WHEN ((TR_retnet#94057175 = NA) OR (TR_retnet#94057175 = null)) THEN null ELSE cast(TR_retnet#94057175 as float) END AS TR_retnet#94057332, CASE WHEN ((IR_retnet#94057176 = NA) OR (IR_retnet#94057176 = null)) THEN null ELSE cast(IR_retnet#94057176 as float) END AS IR_retnet#94057333, CASE WHEN ((turnover#94057177 = NA) OR (turnover#94057177 = null)) THEN null ELSE cast(turnover#94057177 as float) END AS turnover#94057334]
Input [33]: [overall#94057145, cap#94057146, retIC#94057147, resretIC#94057148, size#94057149, value#94057150, growth#94057151, leverage#94057152, volatility#94057153, momentum#94057154, yield#94057155, numcos#94057156, numdates#94057157, annual_bmret#94057158, annual_ret#94057159, std_ret#94057160, Sharpe_ret#94057161, PctPos_ret#94057162, TR_ret#94057163, IR_ret#94057164, annual_resret#94057165, std_resret#94057166, Sharpe_resret#94057167, PctPos_resret#94057168, TR_resret#94057169, IR_resret#94057170, annual_retnet#94057171, std_retnet#94057172, Sharpe_retnet#94057173, PctPos_retnet#94057174, TR_retnet#94057175, IR_retnet#94057176, turnover#94057177]
(7) ColumnarToRow [codegen id : 1]
Input [8]: [cap#94057287, growth#94057296, leverage#94057298, momentum#94057301, size#94057292, value#94057294, volatility#94057299, yield#94057303]
(8) Filter [codegen id : 1]
Input [8]: [cap#94057287, growth#94057296, leverage#94057298, momentum#94057301, size#94057292, value#94057294, volatility#94057299, yield#94057303]
Condition : (isnotnull(cap#94057287) AND (cap#94057287 = 0.0))
(9) Project [codegen id : 1]
Output [7]: [yield#94057303, volatility#94057299, momentum#94057301, size#94057292, value#94057294, growth#94057296, leverage#94057298]
Input [8]: [cap#94057287, growth#94057296, leverage#94057298, momentum#94057301, size#94057292, value#94057294, volatility#94057299, yield#94057303]
(10) ColumnarToRow [codegen id : 1]
Input [7]: [yield#94057303, volatility#94057299, momentum#94057301, size#94057292, value#94057294, growth#94057296, leverage#94057298]
(11) CollectLimit
Input [7]: [yield#94057303, volatility#94057299, momentum#94057301, size#94057292, value#94057294, growth#94057296, leverage#94057298]
Arguments: 1000000