• Submitted Time: 2025/08/02 15:24:09
• Duration: 0.4 s
• Succeeded Jobs: 331452 

== Physical Plan ==
CollectLimit (16)
+- InMemoryTableScan (1)
      +- InMemoryRelation (2)
            +- * Sort (15)
               +- Exchange (14)
                  +- Union (13)
                     :- InMemoryTableScan (3)
                     :     +- InMemoryRelation (4)
                     :           +- * Project (6)
                     :              +- Scan csv  (5)
                     +- * Project (12)
                        +- * Filter (11)
                           +- InMemoryTableScan (7)
                                 +- InMemoryRelation (8)
                                       +- * Project (10)
                                          +- Scan csv  (9)


(1) InMemoryTableScan
Output [2]: [date#94382484, numcos#94382547]
Arguments: [date#94382484, numcos#94382547]

(2) InMemoryRelation
Arguments: [date#94382484, numcos#94382547], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(2) Sort [date#94382484 ASC NULLS FIRST], true, 0
+- Exchange rangepartitioning(date#94382484 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535759]
   +- Union
      :- InMemoryTableScan [date#94382484, numcos#94382547]
      :     +- InMemoryRelation [date#94382484, overall#94382488, ret#94382490, resret#94382518, retnet#94382519, turnover#94382546, numcos#94382547, benchmark#94382548, excess_ret#94382549, excess_resret#94382564, excess_retnet#94382566], StorageLevel(disk, memory, deserialized, 1 replicas)
      :           +- *(1) Project [CASE WHEN (date#94382301 = null) THEN null ELSE cast(date#94382301 as date) END AS date#94382484, CASE WHEN ((overall#94382302 = NA) OR (overall#94382302 = null)) THEN null ELSE cast(overall#94382302 as int) END AS overall#94382488, CASE WHEN ((ret#94382303 = NA) OR (ret#94382303 = null)) THEN null ELSE cast(ret#94382303 as float) END AS ret#94382490, CASE WHEN ((resret#94382304 = NA) OR (resret#94382304 = null)) THEN null ELSE cast(resret#94382304 as float) END AS resret#94382518, CASE WHEN ((retnet#94382305 = NA) OR (retnet#94382305 = null)) THEN null ELSE cast(retnet#94382305 as float) END AS retnet#94382519, CASE WHEN ((turnover#94382306 = NA) OR (turnover#94382306 = null)) THEN null ELSE cast(turnover#94382306 as float) END AS turnover#94382546, CASE WHEN ((numcos#94382307 = NA) OR (numcos#94382307 = null)) THEN null ELSE cast(numcos#94382307 as float) END AS numcos#94382547, CASE WHEN ((benchmark#94382308 = NA) OR (benchmark#94382308 = null)) THEN null ELSE cast(benchmark#94382308 as float) END AS benchmark#94382548, CASE WHEN ((excess_ret#94382309 = NA) OR (excess_ret#94382309 = null)) THEN null ELSE cast(excess_ret#94382309 as float) END AS excess_ret#94382549, CASE WHEN ((excess_resret#94382310 = NA) OR (excess_resret#94382310 = null)) THEN null ELSE cast(excess_resret#94382310 as float) END AS excess_resret#94382564, CASE WHEN ((excess_retnet#94382311 = NA) OR (excess_retnet#94382311 = null)) THEN null ELSE cast(excess_retnet#94382311 as float) END AS excess_retnet#94382566]
      :              +- FileScan csv [date#94382301,overall#94382302,ret#94382303,resret#94382304,retnet#94382305,turnover#94382306,numcos#94382307,benchmark#94382308,excess_ret#94382309,excess_resret#94382310,excess_retnet#94382311] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,overall:string,ret:string,resret:string,retnet:string,turnover:string,numcos:s...
      +- *(1) Project [date#94382455, numcos#94382483]
         +- *(1) Filter (isnotnull(cap#94382456) AND (cast(cap#94382456 as string) = 0))
            +- InMemoryTableScan [cap#94382456, date#94382455, numcos#94382483], [isnotnull(cap#94382456), (cast(cap#94382456 as string) = 0)]
                  +- InMemoryRelation [date#94382455, cap#94382456, ret#94382468, resret#94382469, retnet#94382481, turnover#94382482, numcos#94382483, coverage#94382485, benchmark#94382486, excess_ret#94382487, excess_resret#94382489, excess_retnet#94382517], StorageLevel(disk, memory, deserialized, 1 replicas)
                        +- *(1) Project [CASE WHEN (date#94382274 = null) THEN null ELSE cast(date#94382274 as date) END AS date#94382455, CASE WHEN ((cap#94382275 = NA) OR (cap#94382275 = null)) THEN null ELSE cast(cap#94382275 as float) END AS cap#94382456, CASE WHEN ((ret#94382276 = NA) OR (ret#94382276 = null)) THEN null ELSE cast(ret#94382276 as float) END AS ret#94382468, CASE WHEN ((resret#94382277 = NA) OR (resret#94382277 = null)) THEN null ELSE cast(resret#94382277 as float) END AS resret#94382469, CASE WHEN ((retnet#94382278 = NA) OR (retnet#94382278 = null)) THEN null ELSE cast(retnet#94382278 as float) END AS retnet#94382481, CASE WHEN ((turnover#94382279 = NA) OR (turnover#94382279 = null)) THEN null ELSE cast(turnover#94382279 as float) END AS turnover#94382482, CASE WHEN ((numcos#94382280 = NA) OR (numcos#94382280 = null)) THEN null ELSE cast(numcos#94382280 as float) END AS numcos#94382483, CASE WHEN ((coverage#94382281 = NA) OR (coverage#94382281 = null)) THEN null ELSE cast(coverage#94382281 as float) END AS coverage#94382485, CASE WHEN ((benchmark#94382282 = NA) OR (benchmark#94382282 = null)) THEN null ELSE cast(benchmark#94382282 as float) END AS benchmark#94382486, CASE WHEN ((excess_ret#94382283 = NA) OR (excess_ret#94382283 = null)) THEN null ELSE cast(excess_ret#94382283 as float) END AS excess_ret#94382487, CASE WHEN ((excess_resret#94382284 = NA) OR (excess_resret#94382284 = null)) THEN null ELSE cast(excess_resret#94382284 as float) END AS excess_resret#94382489, CASE WHEN ((excess_retnet#94382285 = NA) OR (excess_retnet#94382285 = null)) THEN null ELSE cast(excess_retnet#94382285 as float) END AS excess_retnet#94382517]
                           +- FileScan csv [date#94382274,cap#94382275,ret#94382276,resret#94382277,retnet#94382278,turnover#94382279,numcos#94382280,coverage#94382281,benchmark#94382282,excess_ret#94382283,excess_resret#94382284,excess_retnet#94382285] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,cap:string,ret:string,resret:string,retnet:string,turnover:string,numcos:strin...
,None), [date#94382484 ASC NULLS FIRST]

(3) InMemoryTableScan
Output [2]: [date#94382484, numcos#94382547]
Arguments: [date#94382484, numcos#94382547]

(4) InMemoryRelation
Arguments: [date#94382484, overall#94382488, ret#94382490, resret#94382518, retnet#94382519, turnover#94382546, numcos#94382547, benchmark#94382548, excess_ret#94382549, excess_resret#94382564, excess_retnet#94382566], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94382301 = null) THEN null ELSE cast(date#94382301 as date) END AS date#94382484, CASE WHEN ((overall#94382302 = NA) OR (overall#94382302 = null)) THEN null ELSE cast(overall#94382302 as int) END AS overall#94382488, CASE WHEN ((ret#94382303 = NA) OR (ret#94382303 = null)) THEN null ELSE cast(ret#94382303 as float) END AS ret#94382490, CASE WHEN ((resret#94382304 = NA) OR (resret#94382304 = null)) THEN null ELSE cast(resret#94382304 as float) END AS resret#94382518, CASE WHEN ((retnet#94382305 = NA) OR (retnet#94382305 = null)) THEN null ELSE cast(retnet#94382305 as float) END AS retnet#94382519, CASE WHEN ((turnover#94382306 = NA) OR (turnover#94382306 = null)) THEN null ELSE cast(turnover#94382306 as float) END AS turnover#94382546, CASE WHEN ((numcos#94382307 = NA) OR (numcos#94382307 = null)) THEN null ELSE cast(numcos#94382307 as float) END AS numcos#94382547, CASE WHEN ((benchmark#94382308 = NA) OR (benchmark#94382308 = null)) THEN null ELSE cast(benchmark#94382308 as float) END AS benchmark#94382548, CASE WHEN ((excess_ret#94382309 = NA) OR (excess_ret#94382309 = null)) THEN null ELSE cast(excess_ret#94382309 as float) END AS excess_ret#94382549, CASE WHEN ((excess_resret#94382310 = NA) OR (excess_resret#94382310 = null)) THEN null ELSE cast(excess_resret#94382310 as float) END AS excess_resret#94382564, CASE WHEN ((excess_retnet#94382311 = NA) OR (excess_retnet#94382311 = null)) THEN null ELSE cast(excess_retnet#94382311 as float) END AS excess_retnet#94382566]
+- FileScan csv [date#94382301,overall#94382302,ret#94382303,resret#94382304,retnet#94382305,turnover#94382306,numcos#94382307,benchmark#94382308,excess_ret#94382309,excess_resret#94382310,excess_retnet#94382311] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,overall:string,ret:string,resret:string,retnet:string,turnover:string,numcos:s...
,None)

(5) Scan csv 
Output [11]: [date#94382301, overall#94382302, ret#94382303, resret#94382304, retnet#94382305, turnover#94382306, numcos#94382307, benchmark#94382308, excess_ret#94382309, excess_resret#94382310, excess_retnet#94382311]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/longshort_overall.csv]
ReadSchema: struct<date:string,overall:string,ret:string,resret:string,retnet:string,turnover:string,numcos:string,benchmark:string,excess_ret:string,excess_resret:string,excess_retnet:string>

(6) Project [codegen id : 1]
Output [11]: [CASE WHEN (date#94382301 = null) THEN null ELSE cast(date#94382301 as date) END AS date#94382484, CASE WHEN ((overall#94382302 = NA) OR (overall#94382302 = null)) THEN null ELSE cast(overall#94382302 as int) END AS overall#94382488, CASE WHEN ((ret#94382303 = NA) OR (ret#94382303 = null)) THEN null ELSE cast(ret#94382303 as float) END AS ret#94382490, CASE WHEN ((resret#94382304 = NA) OR (resret#94382304 = null)) THEN null ELSE cast(resret#94382304 as float) END AS resret#94382518, CASE WHEN ((retnet#94382305 = NA) OR (retnet#94382305 = null)) THEN null ELSE cast(retnet#94382305 as float) END AS retnet#94382519, CASE WHEN ((turnover#94382306 = NA) OR (turnover#94382306 = null)) THEN null ELSE cast(turnover#94382306 as float) END AS turnover#94382546, CASE WHEN ((numcos#94382307 = NA) OR (numcos#94382307 = null)) THEN null ELSE cast(numcos#94382307 as float) END AS numcos#94382547, CASE WHEN ((benchmark#94382308 = NA) OR (benchmark#94382308 = null)) THEN null ELSE cast(benchmark#94382308 as float) END AS benchmark#94382548, CASE WHEN ((excess_ret#94382309 = NA) OR (excess_ret#94382309 = null)) THEN null ELSE cast(excess_ret#94382309 as float) END AS excess_ret#94382549, CASE WHEN ((excess_resret#94382310 = NA) OR (excess_resret#94382310 = null)) THEN null ELSE cast(excess_resret#94382310 as float) END AS excess_resret#94382564, CASE WHEN ((excess_retnet#94382311 = NA) OR (excess_retnet#94382311 = null)) THEN null ELSE cast(excess_retnet#94382311 as float) END AS excess_retnet#94382566]
Input [11]: [date#94382301, overall#94382302, ret#94382303, resret#94382304, retnet#94382305, turnover#94382306, numcos#94382307, benchmark#94382308, excess_ret#94382309, excess_resret#94382310, excess_retnet#94382311]

(7) InMemoryTableScan
Output [3]: [cap#94382456, date#94382455, numcos#94382483]
Arguments: [cap#94382456, date#94382455, numcos#94382483], [isnotnull(cap#94382456), (cast(cap#94382456 as string) = 0)]

(8) InMemoryRelation
Arguments: [date#94382455, cap#94382456, ret#94382468, resret#94382469, retnet#94382481, turnover#94382482, numcos#94382483, coverage#94382485, benchmark#94382486, excess_ret#94382487, excess_resret#94382489, excess_retnet#94382517], CachedRDDBuilder(org.apache.spark.sql.execution.columnar.DefaultCachedBatchSerializer@208e3fd9,StorageLevel(disk, memory, deserialized, 1 replicas),*(1) Project [CASE WHEN (date#94382274 = null) THEN null ELSE cast(date#94382274 as date) END AS date#94382455, CASE WHEN ((cap#94382275 = NA) OR (cap#94382275 = null)) THEN null ELSE cast(cap#94382275 as float) END AS cap#94382456, CASE WHEN ((ret#94382276 = NA) OR (ret#94382276 = null)) THEN null ELSE cast(ret#94382276 as float) END AS ret#94382468, CASE WHEN ((resret#94382277 = NA) OR (resret#94382277 = null)) THEN null ELSE cast(resret#94382277 as float) END AS resret#94382469, CASE WHEN ((retnet#94382278 = NA) OR (retnet#94382278 = null)) THEN null ELSE cast(retnet#94382278 as float) END AS retnet#94382481, CASE WHEN ((turnover#94382279 = NA) OR (turnover#94382279 = null)) THEN null ELSE cast(turnover#94382279 as float) END AS turnover#94382482, CASE WHEN ((numcos#94382280 = NA) OR (numcos#94382280 = null)) THEN null ELSE cast(numcos#94382280 as float) END AS numcos#94382483, CASE WHEN ((coverage#94382281 = NA) OR (coverage#94382281 = null)) THEN null ELSE cast(coverage#94382281 as float) END AS coverage#94382485, CASE WHEN ((benchmark#94382282 = NA) OR (benchmark#94382282 = null)) THEN null ELSE cast(benchmark#94382282 as float) END AS benchmark#94382486, CASE WHEN ((excess_ret#94382283 = NA) OR (excess_ret#94382283 = null)) THEN null ELSE cast(excess_ret#94382283 as float) END AS excess_ret#94382487, CASE WHEN ((excess_resret#94382284 = NA) OR (excess_resret#94382284 = null)) THEN null ELSE cast(excess_resret#94382284 as float) END AS excess_resret#94382489, CASE WHEN ((excess_retnet#94382285 = NA) OR (excess_retnet#94382285 = null)) THEN null ELSE cast(excess_retnet#94382285 as float) END AS excess_retnet#94382517]
+- FileScan csv [date#94382274,cap#94382275,ret#94382276,resret#94382277,retnet#94382278,turnover#94382279,numcos#94382280,coverage#94382281,benchmark#94382282,excess_ret#94382283,excess_resret#94382284,excess_retnet#94382285] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters: [], ReadSchema: struct<date:string,cap:string,ret:string,resret:string,retnet:string,turnover:string,numcos:strin...
,None)

(9) Scan csv 
Output [12]: [date#94382274, cap#94382275, ret#94382276, resret#94382277, retnet#94382278, turnover#94382279, numcos#94382280, coverage#94382281, benchmark#94382282, excess_ret#94382283, excess_resret#94382284, excess_retnet#94382285]
Batched: false
Location: InMemoryFileIndex [file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/longshort_cap.csv]
ReadSchema: struct<date:string,cap:string,ret:string,resret:string,retnet:string,turnover:string,numcos:string,coverage:string,benchmark:string,excess_ret:string,excess_resret:string,excess_retnet:string>

(10) Project [codegen id : 1]
Output [12]: [CASE WHEN (date#94382274 = null) THEN null ELSE cast(date#94382274 as date) END AS date#94382455, CASE WHEN ((cap#94382275 = NA) OR (cap#94382275 = null)) THEN null ELSE cast(cap#94382275 as float) END AS cap#94382456, CASE WHEN ((ret#94382276 = NA) OR (ret#94382276 = null)) THEN null ELSE cast(ret#94382276 as float) END AS ret#94382468, CASE WHEN ((resret#94382277 = NA) OR (resret#94382277 = null)) THEN null ELSE cast(resret#94382277 as float) END AS resret#94382469, CASE WHEN ((retnet#94382278 = NA) OR (retnet#94382278 = null)) THEN null ELSE cast(retnet#94382278 as float) END AS retnet#94382481, CASE WHEN ((turnover#94382279 = NA) OR (turnover#94382279 = null)) THEN null ELSE cast(turnover#94382279 as float) END AS turnover#94382482, CASE WHEN ((numcos#94382280 = NA) OR (numcos#94382280 = null)) THEN null ELSE cast(numcos#94382280 as float) END AS numcos#94382483, CASE WHEN ((coverage#94382281 = NA) OR (coverage#94382281 = null)) THEN null ELSE cast(coverage#94382281 as float) END AS coverage#94382485, CASE WHEN ((benchmark#94382282 = NA) OR (benchmark#94382282 = null)) THEN null ELSE cast(benchmark#94382282 as float) END AS benchmark#94382486, CASE WHEN ((excess_ret#94382283 = NA) OR (excess_ret#94382283 = null)) THEN null ELSE cast(excess_ret#94382283 as float) END AS excess_ret#94382487, CASE WHEN ((excess_resret#94382284 = NA) OR (excess_resret#94382284 = null)) THEN null ELSE cast(excess_resret#94382284 as float) END AS excess_resret#94382489, CASE WHEN ((excess_retnet#94382285 = NA) OR (excess_retnet#94382285 = null)) THEN null ELSE cast(excess_retnet#94382285 as float) END AS excess_retnet#94382517]
Input [12]: [date#94382274, cap#94382275, ret#94382276, resret#94382277, retnet#94382278, turnover#94382279, numcos#94382280, coverage#94382281, benchmark#94382282, excess_ret#94382283, excess_resret#94382284, excess_retnet#94382285]

(11) Filter [codegen id : 1]
Input [3]: [cap#94382456, date#94382455, numcos#94382483]
Condition : (isnotnull(cap#94382456) AND (cast(cap#94382456 as string) = 0))

(12) Project [codegen id : 1]
Output [2]: [date#94382455, numcos#94382483]
Input [3]: [cap#94382456, date#94382455, numcos#94382483]

(13) Union

(14) Exchange
Input [2]: [date#94382484, numcos#94382547]
Arguments: rangepartitioning(date#94382484 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535759]

(15) Sort [codegen id : 2]
Input [2]: [date#94382484, numcos#94382547]
Arguments: [date#94382484 ASC NULLS FIRST], true, 0

(16) CollectLimit
Input [2]: [date#94382484, numcos#94382547]
Arguments: 1000000