Project [fractile#94005101 AS fractile#94005106, annual_ret#94005103 AS annual_ret#94005109, annual_resret#94005104 AS annual_resret#94005115, numcos#94005105 AS numcos#94005116]
Filter (isnotnull(cap#94005102) AND (cast(cap#94005102 as string) = 1))
Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
== Physical Plan ==
CollectLimit (11)
+- * ColumnarToRow (10)
+- InMemoryTableScan (1)
+- InMemoryRelation (2)
+- * Project (9)
+- * Filter (8)
+- * ColumnarToRow (7)
+- InMemoryTableScan (3)
+- InMemoryRelation (4)
+- * Project (6)
+- Scan csv (5)
(1) InMemoryTableScan
Output [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
Arguments: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
(2) InMemoryRelation
Arguments: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [fractile#94005101 AS fractile#94005106, annual_ret#94005103 AS annual_ret#94005109, annual_resret#94005104 AS annual_resret#94005115, numcos#94005105 AS numcos#94005116]
+- *(1) Filter (isnotnull(cap#94005102) AND (cast(cap#94005102 as string) = 1))
+- *(1) ColumnarToRow
+- InMemoryTableScan [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105], [isnotnull(cap#94005102), (cast(cap#94005102 as string) = 1)]
+- InMemoryRelation [fractile#94005101, cap#94005102, annual_ret#94005103, annual_resret#94005104, numcos#94005105], StorageLevel(disk, memory, deserialized, 1 replicas)
+- *(1) Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
+- FileScan csv [fractile#94005058,cap#94005059,annual_ret#94005060,annual_resret#94005061,numcos#94005062] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<fractile:string,cap:string,annual_ret:string,annual_resret:string,numcos:string>
,None)
(3) InMemoryTableScan
Output [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
Arguments: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105], [isnotnull(cap#94005102), (cast(cap#94005102 as string) = 1)]
(4) InMemoryRelation
Arguments: [fractile#94005101, cap#94005102, annual_ret#94005103, annual_resret#94005104, numcos#94005105], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
+- FileScan csv [fractile#94005058,cap#94005059,annual_ret#94005060,annual_resret#94005061,numcos#94005062] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<fractile:string,cap:string,annual_ret:string,annual_resret:string,numcos:string>
,None)
(5) Scan csv
Output [5]: [fractile#94005058, cap#94005059, annual_ret#94005060, annual_resret#94005061, numcos#94005062]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rankviews/fractiles_individually_cap.csv]
ReadSchema: struct<fractile:string,cap:string,annual_ret:string,annual_resret:string,numcos:string>
(6) Project [codegen id : 1]
Output [5]: [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]
Input [5]: [fractile#94005058, cap#94005059, annual_ret#94005060, annual_resret#94005061, numcos#94005062]
(7) ColumnarToRow [codegen id : 1]
Input [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
(8) Filter [codegen id : 1]
Input [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
Condition : (isnotnull(cap#94005102) AND (cast(cap#94005102 as string) = 1))
(9) Project [codegen id : 1]
Output [4]: [fractile#94005101 AS fractile#94005106, annual_ret#94005103 AS annual_ret#94005109, annual_resret#94005104 AS annual_resret#94005115, numcos#94005105 AS numcos#94005116]
Input [5]: [annual_resret#94005104, annual_ret#94005103, cap#94005102, fractile#94005101, numcos#94005105]
(10) ColumnarToRow [codegen id : 1]
Input [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
(11) CollectLimit
Input [4]: [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]
Arguments: 1000000