Project [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(numcos#94005085 as float) END AS numcos#94005116]
Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
== Physical Plan ==
CollectLimit (17)
+- * ColumnarToRow (16)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- Union (15)
:- * ColumnarToRow (7)
: +- InMemoryTableScan (3)
: +- InMemoryRelation (4)
: +- * Project (6)
: +- Scan csv (5)
+- * Project (14)
+- * Filter (13)
+- * ColumnarToRow (12)
+- InMemoryTableScan (8)
+- InMemoryRelation (9)
+- * Project (11)
+- Scan csv (10)
(1) InMemoryTableScan
Output [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
Arguments: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
(2) InMemoryRelation
Arguments: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),Union
:- *(1) ColumnarToRow
: +- InMemoryTableScan [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
: +- InMemoryRelation [fractile#94005106, overall#94005107, annual_ret#94005109, annual_resret#94005115, numcos#94005116], StorageLevel(disk, memory, deserialized, 1 replicas)
: +- *(1) Project [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(numcos#94005085 as float) END AS numcos#94005116]
: +- FileScan csv [fractile#94005081,overall#94005082,annual_ret#94005083,annual_resret#94005084,numcos#94005085] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<fractile:string,overall:string,annual_ret:string,annual_resret:string,numcos:string>
+- *(2) Project [fractile#94005101, annual_ret#94005103, annual_resret#94005104, numcos#94005105]
+- *(2) Filter (isnotnull(cap#94005102) AND (cast(cap#94005102 as string) = 0))
+- *(2) ColumnarToRow
+- InMemoryTableScan [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105], [isnotnull(cap#94005102), (cast(cap#94005102 as string) = 0)]
+- InMemoryRelation [fractile#94005101, cap#94005102, annual_ret#94005103, annual_resret#94005104, numcos#94005105], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
+- FileScan csv [fractile#94005058,cap#94005059,annual_ret#94005060,annual_resret#94005061,numcos#94005062] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<fractile:string,cap:string,annual_ret:string,annual_resret:string,numcos:string>
,None)
(3) InMemoryTableScan
Output [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
Arguments: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
(4) InMemoryRelation
Arguments: [fractile#94005106, overall#94005107, annual_ret#94005109, annual_resret#94005115, numcos#94005116], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(numcos#94005085 as float) END AS numcos#94005116]
+- FileScan csv [fractile#94005081,overall#94005082,annual_ret#94005083,annual_resret#94005084,numcos#94005085] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<fractile:string,overall:string,annual_ret:string,annual_resret:string,numcos:string>
,None)
(5) Scan csv
Output [5]: [fractile#94005081, overall#94005082, annual_ret#94005083, annual_resret#94005084, numcos#94005085]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rankviews/fractiles_individually_overall.csv]
ReadSchema: struct<fractile:string,overall:string,annual_ret:string,annual_resret:string,numcos:string>
(6) Project [codegen id : 1]
Output [5]: [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(numcos#94005085 as float) END AS numcos#94005116]
Input [5]: [fractile#94005081, overall#94005082, annual_ret#94005083, annual_resret#94005084, numcos#94005085]
(7) ColumnarToRow [codegen id : 1]
Input [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
(8) InMemoryTableScan
Output [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
Arguments: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105], [isnotnull(cap#94005102), (cast(cap#94005102 as string) = 0)]
(9) InMemoryRelation
Arguments: [fractile#94005101, cap#94005102, annual_ret#94005103, annual_resret#94005104, numcos#94005105], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
+- FileScan csv [fractile#94005058,cap#94005059,annual_ret#94005060,annual_resret#94005061,numcos#94005062] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<fractile:string,cap:string,annual_ret:string,annual_resret:string,numcos:string>
,None)
(10) Scan csv
Output [5]: [fractile#94005058, cap#94005059, annual_ret#94005060, annual_resret#94005061, numcos#94005062]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rankviews/fractiles_individually_cap.csv]
ReadSchema: struct<fractile:string,cap:string,annual_ret:string,annual_resret:string,numcos:string>
(11) Project [codegen id : 1]
Output [5]: [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
Input [5]: [fractile#94005058, cap#94005059, annual_ret#94005060, annual_resret#94005061, numcos#94005062]
(12) ColumnarToRow [codegen id : 2]
Input [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
(13) Filter [codegen id : 2]
Input [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
Condition : (isnotnull(cap#94005102) AND (cast(cap#94005102 as string) = 0))
(14) Project [codegen id : 2]
Output [4]: [fractile#94005101, annual_ret#94005103, annual_resret#94005104, numcos#94005105]
Input [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
(15) Union
(16) ColumnarToRow [codegen id : 1]
Input [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
(17) CollectLimit
Input [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
Arguments: 1000000