== Physical Plan ==
CollectLimit (10)
+- * ColumnarToRow (9)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (8)
+- * Filter (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [7]: [yield#94014273, volatility#94014269, momentum#94014271, size#94014260, value#94014263, growth#94014265, leverage#94014267]
Arguments: [yield#94014273, volatility#94014269, momentum#94014271, size#94014260, value#94014263, growth#94014265, leverage#94014267]
(2) InMemoryRelation
Arguments: [yield#94014273, volatility#94014269, momentum#94014271, size#94014260, value#94014263, growth#94014265, leverage#94014267], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [yield#94014273, volatility#94014269, momentum#94014271, size#94014260, value#94014263, growth#94014265, leverage#94014267]
+- *(1) Filter (isnotnull(cap#94014254) AND (cap#94014254 = 1.0))
+- InMemoryTableScan [cap#94014254, growth#94014265, leverage#94014267, momentum#94014271, size#94014260, value#94014263, volatility#94014269, yield#94014273], [isnotnull(cap#94014254), (cap#94014254 = 1.0)]
+- InMemoryRelation [overall#94014252, cap#94014254, retIC#94014256, resretIC#94014258, size#94014260, value#94014263, growth#94014265, leverage#94014267, volatility#94014269, momentum#94014271, yield#94014273, numcos#94014276, numdates#94014278, annual_bmret#94014280, annual_ret#94014282, std_ret#94014284, Sharpe_ret#94014286, PctPos_ret#94014288, TR_ret#94014290, IR_ret#94014291, annual_resret#94014293, std_resret#94014294, Sharpe_resret#94014295, PctPos_resret#94014296, ... 9 more fields], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN ((overall#94014066 = NA) OR (overall#94014066 = null)) THEN null ELSE cast(overall#94014066 as int) END AS overall#94014252, CASE WHEN ((cap#94014067 = NA) OR (cap#94014067 = null)) THEN null ELSE cast(cap#94014067 as float) END AS cap#94014254, CASE WHEN ((retIC#94014068 = NA) OR (retIC#94014068 = null)) THEN null ELSE cast(retIC#94014068 as float) END AS retIC#94014256, CASE WHEN ((resretIC#94014069 = NA) OR (resretIC#94014069 = null)) THEN null ELSE cast(resretIC#94014069 as float) END AS resretIC#94014258, CASE WHEN ((size#94014070 = NA) OR (size#94014070 = null)) THEN null ELSE cast(size#94014070 as float) END AS size#94014260, CASE WHEN ((value#94014071 = NA) OR (value#94014071 = null)) THEN null ELSE cast(value#94014071 as float) END AS value#94014263, CASE WHEN ((growth#94014072 = NA) OR (growth#94014072 = null)) THEN null ELSE cast(growth#94014072 as float) END AS growth#94014265, CASE WHEN ((leverage#94014073 = NA) OR (leverage#94014073 = null)) THEN null ELSE cast(leverage#94014073 as float) END AS leverage#94014267, CASE WHEN ((volatility#94014074 = NA) OR (volatility#94014074 = null)) THEN null ELSE cast(volatility#94014074 as float) END AS volatility#94014269, CASE WHEN ((momentum#94014075 = NA) OR (momentum#94014075 = null)) THEN null ELSE cast(momentum#94014075 as float) END AS momentum#94014271, CASE WHEN ((yield#94014076 = NA) OR (yield#94014076 = null)) THEN null ELSE cast(yield#94014076 as float) END AS yield#94014273, CASE WHEN ((numcos#94014077 = NA) OR (numcos#94014077 = null)) THEN null ELSE cast(numcos#94014077 as float) END AS numcos#94014276, CASE WHEN ((numdates#94014078 = NA) OR (numdates#94014078 = null)) THEN null ELSE cast(numdates#94014078 as float) END AS numdates#94014278, CASE WHEN (annual_bmret#94014079 = null) THEN null ELSE annual_bmret#94014079 END AS annual_bmret#94014280, CASE WHEN ((annual_ret#94014080 = NA) OR (annual_ret#94014080 = null)) THEN null ELSE cast(annual_ret#94014080 as float) END AS annual_ret#94014282, CASE WHEN ((std_ret#94014081 = NA) OR (std_ret#94014081 = null)) THEN null ELSE cast(std_ret#94014081 as float) END AS std_ret#94014284, CASE WHEN ((Sharpe_ret#94014082 = NA) OR (Sharpe_ret#94014082 = null)) THEN null ELSE cast(Sharpe_ret#94014082 as float) END AS Sharpe_ret#94014286, CASE WHEN ((PctPos_ret#94014083 = NA) OR (PctPos_ret#94014083 = null)) THEN null ELSE cast(PctPos_ret#94014083 as float) END AS PctPos_ret#94014288, CASE WHEN (TR_ret#94014084 = null) THEN null ELSE TR_ret#94014084 END AS TR_ret#94014290, CASE WHEN (IR_ret#94014085 = null) THEN null ELSE IR_ret#94014085 END AS IR_ret#94014291, CASE WHEN ((annual_resret#94014086 = NA) OR (annual_resret#94014086 = null)) THEN null ELSE cast(annual_resret#94014086 as float) END AS annual_resret#94014293, CASE WHEN ((std_resret#94014087 = NA) OR (std_resret#94014087 = null)) THEN null ELSE cast(std_resret#94014087 as float) END AS std_resret#94014294, CASE WHEN ((Sharpe_resret#94014088 = NA) OR (Sharpe_resret#94014088 = null)) THEN null ELSE cast(Sharpe_resret#94014088 as float) END AS Sharpe_resret#94014295, CASE WHEN ((PctPos_resret#94014089 = NA) OR (PctPos_resret#94014089 = null)) THEN null ELSE cast(PctPos_resret#94014089 as float) END AS PctPos_resret#94014296, ... 9 more fields]
+- FileScan csv [overall#94014066,cap#94014067,retIC#94014068,resretIC#94014069,size#94014070,value#94014071,growth#94014072,leverage#94014073,volatility#94014074,momentum#94014075,yield#94014076,numcos#94014077,numdates#94014078,annual_bmret#94014079,annual_ret#94014080,std_ret#94014081,Sharpe_ret#94014082,PctPos_ret#94014083,TR_ret#94014084,IR_ret#94014085,annual_resret#94014086,std_resret#94014087,Sharpe_resret#94014088,PctPos_resret#94014089,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(3) InMemoryTableScan
Output [8]: [cap#94014254, growth#94014265, leverage#94014267, momentum#94014271, size#94014260, value#94014263, volatility#94014269, yield#94014273]
Arguments: [cap#94014254, growth#94014265, leverage#94014267, momentum#94014271, size#94014260, value#94014263, volatility#94014269, yield#94014273], [isnotnull(cap#94014254), (cap#94014254 = 1.0)]
(4) InMemoryRelation
Arguments: [overall#94014252, cap#94014254, retIC#94014256, resretIC#94014258, size#94014260, value#94014263, growth#94014265, leverage#94014267, volatility#94014269, momentum#94014271, yield#94014273, numcos#94014276, numdates#94014278, annual_bmret#94014280, annual_ret#94014282, std_ret#94014284, Sharpe_ret#94014286, PctPos_ret#94014288, TR_ret#94014290, IR_ret#94014291, annual_resret#94014293, std_resret#94014294, Sharpe_resret#94014295, PctPos_resret#94014296, ... 9 more fields], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((overall#94014066 = NA) OR (overall#94014066 = null)) THEN null ELSE cast(overall#94014066 as int) END AS overall#94014252, CASE WHEN ((cap#94014067 = NA) OR (cap#94014067 = null)) THEN null ELSE cast(cap#94014067 as float) END AS cap#94014254, CASE WHEN ((retIC#94014068 = NA) OR (retIC#94014068 = null)) THEN null ELSE cast(retIC#94014068 as float) END AS retIC#94014256, CASE WHEN ((resretIC#94014069 = NA) OR (resretIC#94014069 = null)) THEN null ELSE cast(resretIC#94014069 as float) END AS resretIC#94014258, CASE WHEN ((size#94014070 = NA) OR (size#94014070 = null)) THEN null ELSE cast(size#94014070 as float) END AS size#94014260, CASE WHEN ((value#94014071 = NA) OR (value#94014071 = null)) THEN null ELSE cast(value#94014071 as float) END AS value#94014263, CASE WHEN ((growth#94014072 = NA) OR (growth#94014072 = null)) THEN null ELSE cast(growth#94014072 as float) END AS growth#94014265, CASE WHEN ((leverage#94014073 = NA) OR (leverage#94014073 = null)) THEN null ELSE cast(leverage#94014073 as float) END AS leverage#94014267, CASE WHEN ((volatility#94014074 = NA) OR (volatility#94014074 = null)) THEN null ELSE cast(volatility#94014074 as float) END AS volatility#94014269, CASE WHEN ((momentum#94014075 = NA) OR (momentum#94014075 = null)) THEN null ELSE cast(momentum#94014075 as float) END AS momentum#94014271, CASE WHEN ((yield#94014076 = NA) OR (yield#94014076 = null)) THEN null ELSE cast(yield#94014076 as float) END AS yield#94014273, CASE WHEN ((numcos#94014077 = NA) OR (numcos#94014077 = null)) THEN null ELSE cast(numcos#94014077 as float) END AS numcos#94014276, CASE WHEN ((numdates#94014078 = NA) OR (numdates#94014078 = null)) THEN null ELSE cast(numdates#94014078 as float) END AS numdates#94014278, CASE WHEN (annual_bmret#94014079 = null) THEN null ELSE annual_bmret#94014079 END AS annual_bmret#94014280, CASE WHEN ((annual_ret#94014080 = NA) OR (annual_ret#94014080 = null)) THEN null ELSE cast(annual_ret#94014080 as float) END AS annual_ret#94014282, CASE WHEN ((std_ret#94014081 = NA) OR (std_ret#94014081 = null)) THEN null ELSE cast(std_ret#94014081 as float) END AS std_ret#94014284, CASE WHEN ((Sharpe_ret#94014082 = NA) OR (Sharpe_ret#94014082 = null)) THEN null ELSE cast(Sharpe_ret#94014082 as float) END AS Sharpe_ret#94014286, CASE WHEN ((PctPos_ret#94014083 = NA) OR (PctPos_ret#94014083 = null)) THEN null ELSE cast(PctPos_ret#94014083 as float) END AS PctPos_ret#94014288, CASE WHEN (TR_ret#94014084 = null) THEN null ELSE TR_ret#94014084 END AS TR_ret#94014290, CASE WHEN (IR_ret#94014085 = null) THEN null ELSE IR_ret#94014085 END AS IR_ret#94014291, CASE WHEN ((annual_resret#94014086 = NA) OR (annual_resret#94014086 = null)) THEN null ELSE cast(annual_resret#94014086 as float) END AS annual_resret#94014293, CASE WHEN ((std_resret#94014087 = NA) OR (std_resret#94014087 = null)) THEN null ELSE cast(std_resret#94014087 as float) END AS std_resret#94014294, CASE WHEN ((Sharpe_resret#94014088 = NA) OR (Sharpe_resret#94014088 = null)) THEN null ELSE cast(Sharpe_resret#94014088 as float) END AS Sharpe_resret#94014295, CASE WHEN ((PctPos_resret#94014089 = NA) OR (PctPos_resret#94014089 = null)) THEN null ELSE cast(PctPos_resret#94014089 as float) END AS PctPos_resret#94014296, ... 9 more fields]
+- FileScan csv [overall#94014066,cap#94014067,retIC#94014068,resretIC#94014069,size#94014070,value#94014071,growth#94014072,leverage#94014073,volatility#94014074,momentum#94014075,yield#94014076,numcos#94014077,numdates#94014078,annual_bmret#94014079,annual_ret#94014080,std_ret#94014081,Sharpe_ret#94014082,PctPos_ret#94014083,TR_ret#94014084,IR_ret#94014085,annual_resret#94014086,std_resret#94014087,Sharpe_resret#94014088,PctPos_resret#94014089,... 9 more fields] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:str...
,None)
(5) Scan csv
Output [33]: [overall#94014066, cap#94014067, retIC#94014068, resretIC#94014069, size#94014070, value#94014071, growth#94014072, leverage#94014073, volatility#94014074, momentum#94014075, yield#94014076, numcos#94014077, numdates#94014078, annual_bmret#94014079, annual_ret#94014080, std_ret#94014081, Sharpe_ret#94014082, PctPos_ret#94014083, TR_ret#94014084, IR_ret#94014085, annual_resret#94014086, std_resret#94014087, Sharpe_resret#94014088, PctPos_resret#94014089, TR_resret#94014090, IR_resret#94014091, annual_retnet#94014092, std_retnet#94014093, Sharpe_retnet#94014094, PctPos_retnet#94014095, TR_retnet#94014096, IR_retnet#94014097, turnover#94014098]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rankviews/stats_overall.csv]
ReadSchema: struct<overall:string,cap:string,retIC:string,resretIC:string,size:string,value:string,growth:string,leverage:string,volatility:string,momentum:string,yield:string,numcos:string,numdates:string,annual_bmret:string,annual_ret:string,std_ret:string,Sharpe_ret:string,PctPos_ret:string,TR_ret:string,IR_ret:string,annual_resret:string,std_resret:string,Sharpe_resret:string,PctPos_resret:string,TR_resret:string,IR_resret:string,annual_retnet:string,std_retnet:string,Sharpe_retnet:string,PctPos_retnet:string,TR_retnet:string,IR_retnet:string,turnover:string>
(6) Project [codegen id : 1]
Output [33]: [CASE WHEN ((overall#94014066 = NA) OR (overall#94014066 = null)) THEN null ELSE cast(overall#94014066 as int) END AS overall#94014252, CASE WHEN ((cap#94014067 = NA) OR (cap#94014067 = null)) THEN null ELSE cast(cap#94014067 as float) END AS cap#94014254, CASE WHEN ((retIC#94014068 = NA) OR (retIC#94014068 = null)) THEN null ELSE cast(retIC#94014068 as float) END AS retIC#94014256, CASE WHEN ((resretIC#94014069 = NA) OR (resretIC#94014069 = null)) THEN null ELSE cast(resretIC#94014069 as float) END AS resretIC#94014258, CASE WHEN ((size#94014070 = NA) OR (size#94014070 = null)) THEN null ELSE cast(size#94014070 as float) END AS size#94014260, CASE WHEN ((value#94014071 = NA) OR (value#94014071 = null)) THEN null ELSE cast(value#94014071 as float) END AS value#94014263, CASE WHEN ((growth#94014072 = NA) OR (growth#94014072 = null)) THEN null ELSE cast(growth#94014072 as float) END AS growth#94014265, CASE WHEN ((leverage#94014073 = NA) OR (leverage#94014073 = null)) THEN null ELSE cast(leverage#94014073 as float) END AS leverage#94014267, CASE WHEN ((volatility#94014074 = NA) OR (volatility#94014074 = null)) THEN null ELSE cast(volatility#94014074 as float) END AS volatility#94014269, CASE WHEN ((momentum#94014075 = NA) OR (momentum#94014075 = null)) THEN null ELSE cast(momentum#94014075 as float) END AS momentum#94014271, CASE WHEN ((yield#94014076 = NA) OR (yield#94014076 = null)) THEN null ELSE cast(yield#94014076 as float) END AS yield#94014273, CASE WHEN ((numcos#94014077 = NA) OR (numcos#94014077 = null)) THEN null ELSE cast(numcos#94014077 as float) END AS numcos#94014276, CASE WHEN ((numdates#94014078 = NA) OR (numdates#94014078 = null)) THEN null ELSE cast(numdates#94014078 as float) END AS numdates#94014278, CASE WHEN (annual_bmret#94014079 = null) THEN null ELSE annual_bmret#94014079 END AS annual_bmret#94014280, CASE WHEN ((annual_ret#94014080 = NA) OR (annual_ret#94014080 = null)) THEN null ELSE cast(annual_ret#94014080 as float) END AS annual_ret#94014282, CASE WHEN ((std_ret#94014081 = NA) OR (std_ret#94014081 = null)) THEN null ELSE cast(std_ret#94014081 as float) END AS std_ret#94014284, CASE WHEN ((Sharpe_ret#94014082 = NA) OR (Sharpe_ret#94014082 = null)) THEN null ELSE cast(Sharpe_ret#94014082 as float) END AS Sharpe_ret#94014286, CASE WHEN ((PctPos_ret#94014083 = NA) OR (PctPos_ret#94014083 = null)) THEN null ELSE cast(PctPos_ret#94014083 as float) END AS PctPos_ret#94014288, CASE WHEN (TR_ret#94014084 = null) THEN null ELSE TR_ret#94014084 END AS TR_ret#94014290, CASE WHEN (IR_ret#94014085 = null) THEN null ELSE IR_ret#94014085 END AS IR_ret#94014291, CASE WHEN ((annual_resret#94014086 = NA) OR (annual_resret#94014086 = null)) THEN null ELSE cast(annual_resret#94014086 as float) END AS annual_resret#94014293, CASE WHEN ((std_resret#94014087 = NA) OR (std_resret#94014087 = null)) THEN null ELSE cast(std_resret#94014087 as float) END AS std_resret#94014294, CASE WHEN ((Sharpe_resret#94014088 = NA) OR (Sharpe_resret#94014088 = null)) THEN null ELSE cast(Sharpe_resret#94014088 as float) END AS Sharpe_resret#94014295, CASE WHEN ((PctPos_resret#94014089 = NA) OR (PctPos_resret#94014089 = null)) THEN null ELSE cast(PctPos_resret#94014089 as float) END AS PctPos_resret#94014296, CASE WHEN (TR_resret#94014090 = null) THEN null ELSE TR_resret#94014090 END AS TR_resret#94014297, CASE WHEN (IR_resret#94014091 = null) THEN null ELSE IR_resret#94014091 END AS IR_resret#94014298, CASE WHEN ((annual_retnet#94014092 = NA) OR (annual_retnet#94014092 = null)) THEN null ELSE cast(annual_retnet#94014092 as float) END AS annual_retnet#94014299, CASE WHEN ((std_retnet#94014093 = NA) OR (std_retnet#94014093 = null)) THEN null ELSE cast(std_retnet#94014093 as float) END AS std_retnet#94014300, CASE WHEN ((Sharpe_retnet#94014094 = NA) OR (Sharpe_retnet#94014094 = null)) THEN null ELSE cast(Sharpe_retnet#94014094 as float) END AS Sharpe_retnet#94014301, CASE WHEN ((PctPos_retnet#94014095 = NA) OR (PctPos_retnet#94014095 = null)) THEN null ELSE cast(PctPos_retnet#94014095 as float) END AS PctPos_retnet#94014302, CASE WHEN (TR_retnet#94014096 = null) THEN null ELSE TR_retnet#94014096 END AS TR_retnet#94014303, CASE WHEN (IR_retnet#94014097 = null) THEN null ELSE IR_retnet#94014097 END AS IR_retnet#94014304, CASE WHEN ((turnover#94014098 = NA) OR (turnover#94014098 = null)) THEN null ELSE cast(turnover#94014098 as float) END AS turnover#94014305]
Input [33]: [overall#94014066, cap#94014067, retIC#94014068, resretIC#94014069, size#94014070, value#94014071, growth#94014072, leverage#94014073, volatility#94014074, momentum#94014075, yield#94014076, numcos#94014077, numdates#94014078, annual_bmret#94014079, annual_ret#94014080, std_ret#94014081, Sharpe_ret#94014082, PctPos_ret#94014083, TR_ret#94014084, IR_ret#94014085, annual_resret#94014086, std_resret#94014087, Sharpe_resret#94014088, PctPos_resret#94014089, TR_resret#94014090, IR_resret#94014091, annual_retnet#94014092, std_retnet#94014093, Sharpe_retnet#94014094, PctPos_retnet#94014095, TR_retnet#94014096, IR_retnet#94014097, turnover#94014098]
(7) Filter [codegen id : 1]
Input [8]: [cap#94014254, growth#94014265, leverage#94014267, momentum#94014271, size#94014260, value#94014263, volatility#94014269, yield#94014273]
Condition : (isnotnull(cap#94014254) AND (cap#94014254 = 1.0))
(8) Project [codegen id : 1]
Output [7]: [yield#94014273, volatility#94014269, momentum#94014271, size#94014260, value#94014263, growth#94014265, leverage#94014267]
Input [8]: [cap#94014254, growth#94014265, leverage#94014267, momentum#94014271, size#94014260, value#94014263, volatility#94014269, yield#94014273]
(9) ColumnarToRow [codegen id : 1]
Input [7]: [yield#94014273, volatility#94014269, momentum#94014271, size#94014260, value#94014263, growth#94014265, leverage#94014267]
(10) CollectLimit
Input [7]: [yield#94014273, volatility#94014269, momentum#94014271, size#94014260, value#94014263, growth#94014265, leverage#94014267]
Arguments: 1000000