== Physical Plan ==
CollectLimit (11)
+- * ColumnarToRow (10)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (9)
+- * Filter (8)
+- * ColumnarToRow (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [7]: [yield#94071413, volatility#94071409, momentum#94071411, size#94071401, value#94071403, growth#94071405, leverage#94071407]
Arguments: [yield#94071413, volatility#94071409, momentum#94071411, size#94071401, value#94071403, growth#94071405, leverage#94071407]
(2) InMemoryRelation
Arguments: [yield#94071413, volatility#94071409, momentum#94071411, size#94071401, value#94071403, growth#94071405, leverage#94071407], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [yield#94071413, volatility#94071409, momentum#94071411, size#94071401, value#94071403, growth#94071405, leverage#94071407]
+- *(1) Filter (isnotnull(cap#94071395) AND (cap#94071395 = 0.0))
+- *(1) ColumnarToRow
+- InMemoryTableScan [cap#94071395, growth#94071405, leverage#94071407, momentum#94071411, size#94071401, value#94071403, volatility#94071409, yield#94071413], [isnotnull(cap#94071395), (cap#94071395 = 0.0)]
+- InMemoryRelation [overall#94071393, cap#94071395, retIC#94071397, resretIC#94071399, size#94071401, value#94071403, growth#94071405, leverage#94071407, volatility#94071409, momentum#94071411, yield#94071413, numcos#94071415, numdates#94071417, annual_bmret#94071419, annual_ret#94071421, std_ret#94071423, Sharpe_ret#94071425, PctPos_ret#94071427, TR_ret#94071429, IR_ret#94071431, annual_resret#94071433, std_resret#94071435, Sharpe_resret#94071437, PctPos_resret#94071439, ... 9 more fields], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN ((overall#94071160 = NA) OR (overall#94071160 = null)) THEN null ELSE cast(overall#94071160 as int) END AS overall#94071393, CASE WHEN ((cap#94071161 = NA) OR (cap#94071161 = null)) THEN null ELSE cast(cap#94071161 as float) END AS cap#94071395, CASE WHEN ((retIC#94071162 = NA) OR (retIC#94071162 = null)) THEN null ELSE cast(retIC#94071162 as float) END AS retIC#94071397, CASE WHEN ((resretIC#94071163 = NA) OR (resretIC#94071163 = null)) THEN null ELSE cast(resretIC#94071163 as float) END AS resretIC#94071399, CASE WHEN ((size#94071164 = NA) OR (size#94071164 = null)) THEN null ELSE cast(size#94071164 as float) END AS size#94071401, CASE WHEN ((value#94071165 = NA) OR (value#94071165 = null)) THEN null ELSE cast(value#94071165 as float) END AS value#94071403, CASE WHEN ((growth#94071166 = NA) OR (growth#94071166 = null)) THEN null ELSE cast(growth#94071166 as float) END AS growth#94071405, CASE WHEN ((leverage#94071167 = NA) OR (leverage#94071167 = null)) THEN null ELSE cast(leverage#94071167 as float) END AS leverage#94071407, CASE WHEN ((volatility#94071168 = NA) OR (volatility#94071168 = null)) THEN null ELSE cast(volatility#94071168 as float) END AS volatility#94071409, CASE WHEN ((momentum#94071169 = NA) OR (momentum#94071169 = null)) THEN null ELSE cast(momentum#94071169 as float) END AS momentum#94071411, CASE WHEN ((yield#94071170 = NA) OR (yield#94071170 = null)) THEN null ELSE cast(yield#94071170 as float) END AS yield#94071413, CASE WHEN ((numcos#94071171 = NA) OR (numcos#94071171 = null)) THEN null ELSE cast(numcos#94071171 as float) END AS numcos#94071415, CASE WHEN ((numdates#94071172 = NA) OR (numdates#94071172 = null)) THEN null ELSE cast(numdates#94071172 as float) END AS numdates#94071417, CASE WHEN ((annual_bmret#94071173 = NA) OR (annual_bmret#94071173 = null)) THEN null ELSE cast(annual_bmret#94071173 as float) END AS annual_bmret#94071419, CASE WHEN ((annual_ret#94071174 = NA) OR (annual_ret#94071174 = null)) THEN null ELSE cast(annual_ret#94071174 as float) END AS annual_ret#94071421, CASE WHEN ((std_ret#94071175 = NA) OR (std_ret#94071175 = null)) THEN null ELSE cast(std_ret#94071175 as float) END AS std_ret#94071423, CASE WHEN ((Sharpe_ret#94071176 = NA) OR (Sharpe_ret#94071176 = null)) THEN null ELSE cast(Sharpe_ret#94071176 as float) END AS Sharpe_ret#94071425, CASE WHEN ((PctPos_ret#94071177 = NA) OR (PctPos_ret#94071177 = null)) THEN null ELSE cast(PctPos_ret#94071177 as float) END AS PctPos_ret#94071427, CASE WHEN ((TR_ret#94071178 = NA) OR (TR_ret#94071178 = null)) THEN null ELSE cast(TR_ret#94071178 as float) END AS TR_ret#94071429, CASE WHEN ((IR_ret#94071179 = NA) OR (IR_ret#94071179 = null)) THEN null ELSE cast(IR_ret#94071179 as float) END AS IR_ret#94071431, CASE WHEN ((annual_resret#94071180 = NA) OR (annual_resret#94071180 = null)) THEN null ELSE cast(annual_resret#94071180 as float) END AS annual_resret#94071433, CASE WHEN ((std_resret#94071181 = NA) OR (std_resret#94071181 = null)) THEN null ELSE cast(std_resret#94071181 as float) END AS std_resret#94071435, CASE WHEN ((Sharpe_resret#94071182 = NA) OR (Sharpe_resret#94071182 = null)) THEN null ELSE cast(Sharpe_resret#94071182 as float) END AS Sharpe_resret#94071437, CASE WHEN ((PctPos_resret#94071183 = NA) OR (PctPos_resret#94071183 = null)) THEN null ELSE cast(PctPos_resret#94071183 as float) END AS PctPos_resret#94071439, ... 9 more fields]
+- FileScan csv [overall#94071160,cap#94071161,retIC#94071162,resretIC#94071163,size#94071164,value#94071165,growth#94071166,leverage#94071167,volatility#94071168,momentum#94071169,yield#94071170,numcos#94071171,numdates#94071172,annual_bmret#94071173,annual_ret#94071174,std_ret#94071175,Sharpe_ret#94071176,PctPos_ret#94071177,TR_ret#94071178,IR_ret#94071179,annual_resret#94071180,std_resret#94071181,Sharpe_resret#94071182,PctPos_resret#94071183,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/value/sta..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(3) InMemoryTableScan
Output [8]: [cap#94071395, growth#94071405, leverage#94071407, momentum#94071411, size#94071401, value#94071403, volatility#94071409, yield#94071413]
Arguments: [cap#94071395, growth#94071405, leverage#94071407, momentum#94071411, size#94071401, value#94071403, volatility#94071409, yield#94071413], [isnotnull(cap#94071395), (cap#94071395 = 0.0)]
(4) InMemoryRelation
Arguments: [overall#94071393, cap#94071395, retIC#94071397, resretIC#94071399, size#94071401, value#94071403, growth#94071405, leverage#94071407, volatility#94071409, momentum#94071411, yield#94071413, numcos#94071415, numdates#94071417, annual_bmret#94071419, annual_ret#94071421, std_ret#94071423, Sharpe_ret#94071425, PctPos_ret#94071427, TR_ret#94071429, IR_ret#94071431, annual_resret#94071433, std_resret#94071435, Sharpe_resret#94071437, PctPos_resret#94071439, ... 9 more fields], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((overall#94071160 = NA) OR (overall#94071160 = null)) THEN null ELSE cast(overall#94071160 as int) END AS overall#94071393, CASE WHEN ((cap#94071161 = NA) OR (cap#94071161 = null)) THEN null ELSE cast(cap#94071161 as float) END AS cap#94071395, CASE WHEN ((retIC#94071162 = NA) OR (retIC#94071162 = null)) THEN null ELSE cast(retIC#94071162 as float) END AS retIC#94071397, CASE WHEN ((resretIC#94071163 = NA) OR (resretIC#94071163 = null)) THEN null ELSE cast(resretIC#94071163 as float) END AS resretIC#94071399, CASE WHEN ((size#94071164 = NA) OR (size#94071164 = null)) THEN null ELSE cast(size#94071164 as float) END AS size#94071401, CASE WHEN ((value#94071165 = NA) OR (value#94071165 = null)) THEN null ELSE cast(value#94071165 as float) END AS value#94071403, CASE WHEN ((growth#94071166 = NA) OR (growth#94071166 = null)) THEN null ELSE cast(growth#94071166 as float) END AS growth#94071405, CASE WHEN ((leverage#94071167 = NA) OR (leverage#94071167 = null)) THEN null ELSE cast(leverage#94071167 as float) END AS leverage#94071407, CASE WHEN ((volatility#94071168 = NA) OR (volatility#94071168 = null)) THEN null ELSE cast(volatility#94071168 as float) END AS volatility#94071409, CASE WHEN ((momentum#94071169 = NA) OR (momentum#94071169 = null)) THEN null ELSE cast(momentum#94071169 as float) END AS momentum#94071411, CASE WHEN ((yield#94071170 = NA) OR (yield#94071170 = null)) THEN null ELSE cast(yield#94071170 as float) END AS yield#94071413, CASE WHEN ((numcos#94071171 = NA) OR (numcos#94071171 = null)) THEN null ELSE cast(numcos#94071171 as float) END AS numcos#94071415, CASE WHEN ((numdates#94071172 = NA) OR (numdates#94071172 = null)) THEN null ELSE cast(numdates#94071172 as float) END AS numdates#94071417, CASE WHEN ((annual_bmret#94071173 = NA) OR (annual_bmret#94071173 = null)) THEN null ELSE cast(annual_bmret#94071173 as float) END AS annual_bmret#94071419, CASE WHEN ((annual_ret#94071174 = NA) OR (annual_ret#94071174 = null)) THEN null ELSE cast(annual_ret#94071174 as float) END AS annual_ret#94071421, CASE WHEN ((std_ret#94071175 = NA) OR (std_ret#94071175 = null)) THEN null ELSE cast(std_ret#94071175 as float) END AS std_ret#94071423, CASE WHEN ((Sharpe_ret#94071176 = NA) OR (Sharpe_ret#94071176 = null)) THEN null ELSE cast(Sharpe_ret#94071176 as float) END AS Sharpe_ret#94071425, CASE WHEN ((PctPos_ret#94071177 = NA) OR (PctPos_ret#94071177 = null)) THEN null ELSE cast(PctPos_ret#94071177 as float) END AS PctPos_ret#94071427, CASE WHEN ((TR_ret#94071178 = NA) OR (TR_ret#94071178 = null)) THEN null ELSE cast(TR_ret#94071178 as float) END AS TR_ret#94071429, CASE WHEN ((IR_ret#94071179 = NA) OR (IR_ret#94071179 = null)) THEN null ELSE cast(IR_ret#94071179 as float) END AS IR_ret#94071431, CASE WHEN ((annual_resret#94071180 = NA) OR (annual_resret#94071180 = null)) THEN null ELSE cast(annual_resret#94071180 as float) END AS annual_resret#94071433, CASE WHEN ((std_resret#94071181 = NA) OR (std_resret#94071181 = null)) THEN null ELSE cast(std_resret#94071181 as float) END AS std_resret#94071435, CASE WHEN ((Sharpe_resret#94071182 = NA) OR (Sharpe_resret#94071182 = null)) THEN null ELSE cast(Sharpe_resret#94071182 as float) END AS Sharpe_resret#94071437, CASE WHEN ((PctPos_resret#94071183 = NA) OR (PctPos_resret#94071183 = null)) THEN null ELSE cast(PctPos_resret#94071183 as float) END AS PctPos_resret#94071439, ... 9 more fields]
+- FileScan csv [overall#94071160,cap#94071161,retIC#94071162,resretIC#94071163,size#94071164,value#94071165,growth#94071166,leverage#94071167,volatility#94071168,momentum#94071169,yield#94071170,numcos#94071171,numdates#94071172,annual_bmret#94071173,annual_ret#94071174,std_ret#94071175,Sharpe_ret#94071176,PctPos_ret#94071177,TR_ret#94071178,IR_ret#94071179,annual_resret#94071180,std_resret#94071181,Sharpe_resret#94071182,PctPos_resret#94071183,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/value/sta..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(5) Scan csv
Output [33]: [overall#94071160, cap#94071161, retIC#94071162, resretIC#94071163, size#94071164, value#94071165, growth#94071166, leverage#94071167, volatility#94071168, momentum#94071169, yield#94071170, numcos#94071171, numdates#94071172, annual_bmret#94071173, annual_ret#94071174, std_ret#94071175, Sharpe_ret#94071176, PctPos_ret#94071177, TR_ret#94071178, IR_ret#94071179, annual_resret#94071180, std_resret#94071181, Sharpe_resret#94071182, PctPos_resret#94071183, TR_resret#94071184, IR_resret#94071185, annual_retnet#94071186, std_retnet#94071187, Sharpe_retnet#94071188, PctPos_retnet#94071189, TR_retnet#94071190, IR_retnet#94071191, turnover#94071192]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/value/stats_overall.csv]
ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:string,leverage:string,volatility:string,momentum:string,yield:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string>
(6) Project [codegen id : 1]
Output [33]: [CASE WHEN ((overall#94071160 = NA) OR (overall#94071160 = null)) THEN null ELSE cast(overall#94071160 as int) END AS overall#94071393, CASE WHEN ((cap#94071161 = NA) OR (cap#94071161 = null)) THEN null ELSE cast(cap#94071161 as float) END AS cap#94071395, CASE WHEN ((retIC#94071162 = NA) OR (retIC#94071162 = null)) THEN null ELSE cast(retIC#94071162 as float) END AS retIC#94071397, CASE WHEN ((resretIC#94071163 = NA) OR (resretIC#94071163 = null)) THEN null ELSE cast(resretIC#94071163 as float) END AS resretIC#94071399, CASE WHEN ((size#94071164 = NA) OR (size#94071164 = null)) THEN null ELSE cast(size#94071164 as float) END AS size#94071401, CASE WHEN ((value#94071165 = NA) OR (value#94071165 = null)) THEN null ELSE cast(value#94071165 as float) END AS value#94071403, CASE WHEN ((growth#94071166 = NA) OR (growth#94071166 = null)) THEN null ELSE cast(growth#94071166 as float) END AS growth#94071405, CASE WHEN ((leverage#94071167 = NA) OR (leverage#94071167 = null)) THEN null ELSE cast(leverage#94071167 as float) END AS leverage#94071407, CASE WHEN ((volatility#94071168 = NA) OR (volatility#94071168 = null)) THEN null ELSE cast(volatility#94071168 as float) END AS volatility#94071409, CASE WHEN ((momentum#94071169 = NA) OR (momentum#94071169 = null)) THEN null ELSE cast(momentum#94071169 as float) END AS momentum#94071411, CASE WHEN ((yield#94071170 = NA) OR (yield#94071170 = null)) THEN null ELSE cast(yield#94071170 as float) END AS yield#94071413, CASE WHEN ((numcos#94071171 = NA) OR (numcos#94071171 = null)) THEN null ELSE cast(numcos#94071171 as float) END AS numcos#94071415, CASE WHEN ((numdates#94071172 = NA) OR (numdates#94071172 = null)) THEN null ELSE cast(numdates#94071172 as float) END AS numdates#94071417, CASE WHEN ((annual_bmret#94071173 = NA) OR (annual_bmret#94071173 = null)) THEN null ELSE cast(annual_bmret#94071173 as float) END AS annual_bmret#94071419, CASE WHEN ((annual_ret#94071174 = NA) OR (annual_ret#94071174 = null)) THEN null ELSE cast(annual_ret#94071174 as float) END AS annual_ret#94071421, CASE WHEN ((std_ret#94071175 = NA) OR (std_ret#94071175 = null)) THEN null ELSE cast(std_ret#94071175 as float) END AS std_ret#94071423, CASE WHEN ((Sharpe_ret#94071176 = NA) OR (Sharpe_ret#94071176 = null)) THEN null ELSE cast(Sharpe_ret#94071176 as float) END AS Sharpe_ret#94071425, CASE WHEN ((PctPos_ret#94071177 = NA) OR (PctPos_ret#94071177 = null)) THEN null ELSE cast(PctPos_ret#94071177 as float) END AS PctPos_ret#94071427, CASE WHEN ((TR_ret#94071178 = NA) OR (TR_ret#94071178 = null)) THEN null ELSE cast(TR_ret#94071178 as float) END AS TR_ret#94071429, CASE WHEN ((IR_ret#94071179 = NA) OR (IR_ret#94071179 = null)) THEN null ELSE cast(IR_ret#94071179 as float) END AS IR_ret#94071431, CASE WHEN ((annual_resret#94071180 = NA) OR (annual_resret#94071180 = null)) THEN null ELSE cast(annual_resret#94071180 as float) END AS annual_resret#94071433, CASE WHEN ((std_resret#94071181 = NA) OR (std_resret#94071181 = null)) THEN null ELSE cast(std_resret#94071181 as float) END AS std_resret#94071435, CASE WHEN ((Sharpe_resret#94071182 = NA) OR (Sharpe_resret#94071182 = null)) THEN null ELSE cast(Sharpe_resret#94071182 as float) END AS Sharpe_resret#94071437, CASE WHEN ((PctPos_resret#94071183 = NA) OR (PctPos_resret#94071183 = null)) THEN null ELSE cast(PctPos_resret#94071183 as float) END AS PctPos_resret#94071439, CASE WHEN ((TR_resret#94071184 = NA) OR (TR_resret#94071184 = null)) THEN null ELSE cast(TR_resret#94071184 as float) END AS TR_resret#94071441, CASE WHEN ((IR_resret#94071185 = NA) OR (IR_resret#94071185 = null)) THEN null ELSE cast(IR_resret#94071185 as float) END AS IR_resret#94071442, CASE WHEN ((annual_retnet#94071186 = NA) OR (annual_retnet#94071186 = null)) THEN null ELSE cast(annual_retnet#94071186 as float) END AS annual_retnet#94071443, CASE WHEN ((std_retnet#94071187 = NA) OR (std_retnet#94071187 = null)) THEN null ELSE cast(std_retnet#94071187 as float) END AS std_retnet#94071444, CASE WHEN ((Sharpe_retnet#94071188 = NA) OR (Sharpe_retnet#94071188 = null)) THEN null ELSE cast(Sharpe_retnet#94071188 as float) END AS Sharpe_retnet#94071445, CASE WHEN ((PctPos_retnet#94071189 = NA) OR (PctPos_retnet#94071189 = null)) THEN null ELSE cast(PctPos_retnet#94071189 as float) END AS PctPos_retnet#94071446, CASE WHEN ((TR_retnet#94071190 = NA) OR (TR_retnet#94071190 = null)) THEN null ELSE cast(TR_retnet#94071190 as float) END AS TR_retnet#94071447, CASE WHEN ((IR_retnet#94071191 = NA) OR (IR_retnet#94071191 = null)) THEN null ELSE cast(IR_retnet#94071191 as float) END AS IR_retnet#94071448, CASE WHEN ((turnover#94071192 = NA) OR (turnover#94071192 = null)) THEN null ELSE cast(turnover#94071192 as float) END AS turnover#94071449]
Input [33]: [overall#94071160, cap#94071161, retIC#94071162, resretIC#94071163, size#94071164, value#94071165, growth#94071166, leverage#94071167, volatility#94071168, momentum#94071169, yield#94071170, numcos#94071171, numdates#94071172, annual_bmret#94071173, annual_ret#94071174, std_ret#94071175, Sharpe_ret#94071176, PctPos_ret#94071177, TR_ret#94071178, IR_ret#94071179, annual_resret#94071180, std_resret#94071181, Sharpe_resret#94071182, PctPos_resret#94071183, TR_resret#94071184, IR_resret#94071185, annual_retnet#94071186, std_retnet#94071187, Sharpe_retnet#94071188, PctPos_retnet#94071189, TR_retnet#94071190, IR_retnet#94071191, turnover#94071192]
(7) ColumnarToRow [codegen id : 1]
Input [8]: [cap#94071395, growth#94071405, leverage#94071407, momentum#94071411, size#94071401, value#94071403, volatility#94071409, yield#94071413]
(8) Filter [codegen id : 1]
Input [8]: [cap#94071395, growth#94071405, leverage#94071407, momentum#94071411, size#94071401, value#94071403, volatility#94071409, yield#94071413]
Condition : (isnotnull(cap#94071395) AND (cap#94071395 = 0.0))
(9) Project [codegen id : 1]
Output [7]: [yield#94071413, volatility#94071409, momentum#94071411, size#94071401, value#94071403, growth#94071405, leverage#94071407]
Input [8]: [cap#94071395, growth#94071405, leverage#94071407, momentum#94071411, size#94071401, value#94071403, volatility#94071409, yield#94071413]
(10) ColumnarToRow [codegen id : 1]
Input [7]: [yield#94071413, volatility#94071409, momentum#94071411, size#94071401, value#94071403, growth#94071405, leverage#94071407]
(11) CollectLimit
Input [7]: [yield#94071413, volatility#94071409, momentum#94071411, size#94071401, value#94071403, growth#94071405, leverage#94071407]
Arguments: 1000000