PlusAMP App Middleware - Details for Job 331390

Status: SUCCEEDED
Submitted: 2025/08/02 15:21:59
Duration: 55 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_512951 { label="Stage 512951"; subgraph cluster6301857 { label="Scan csv "; 3624019 [labelType="html" label="FileScanRDD [3624019] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624020 [labelType="html" label="MapPartitionsRDD [3624020] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301854 { label="WholeStageCodegen (1)"; 3624021 [labelType="html" label="MapPartitionsRDD [3624021] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301853 { label="InMemoryTableScan"; 3624022 [labelType="html" label="MapPartitionsRDD [3624022] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624023 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94362738 = NA) OR (fractile#94362738 = null)) THEN null ELSE cast(fractile#94362738 as float) END AS fractile#94362813, CASE WHEN ((overall#94362739 = NA) OR (overall#94362739 = null)) THEN null ELSE cast(overall#94362739 as int) END AS overall#94362814, CASE WHEN ((annual_ret#94362740 = NA) OR (annual_ret#94362740 = null)) THEN null ELSE cast(annual_ret#94362740 as float) END AS annual_ret#94362815, CASE WHEN ((annual_resret#94362741 = NA) OR (annual_resret#94362741 = null)) THEN null ELSE cast(annual_resret#94362741 as float) END AS annual_resret#94362827, CASE WHEN ((numcos#94362742 = NA) OR (numcos#94362742 = null)) THEN null ELSE cast(numcos#94362742 as float) END AS numcos#94362836]\n+- FileScan csv [fractile#94362738,overall#94362739,annual_ret#94362740,annual_resret#94362741,numcos#94362742] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/fr..., PartitionFilters: ... [3624023] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624024 [labelType="html" label="MapPartitionsRDD [3624024] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624025 [labelType="html" label="MapPartitionsRDD [3624025] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624026 [labelType="html" label="MapPartitionsRDD [3624026] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301850 { label="WholeStageCodegen (1)"; 3624027 [labelType="html" label="MapPartitionsRDD [3624027] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301867 { label="Scan csv "; 3624028 [labelType="html" label="FileScanRDD [3624028] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624029 [labelType="html" label="MapPartitionsRDD [3624029] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301864 { label="WholeStageCodegen (1)"; 3624030 [labelType="html" label="MapPartitionsRDD [3624030] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301863 { label="InMemoryTableScan"; 3624031 [labelType="html" label="MapPartitionsRDD [3624031] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624032 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94362530 = NA) OR (fractile#94362530 = null)) THEN null ELSE cast(fractile#94362530 as float) END AS fractile#94362552, CASE WHEN ((cap#94362531 = NA) OR (cap#94362531 = null)) THEN null ELSE cast(cap#94362531 as float) END AS cap#94362553, CASE WHEN ((annual_ret#94362532 = NA) OR (annual_ret#94362532 = null)) THEN null ELSE cast(annual_ret#94362532 as float) END AS annual_ret#94362554, CASE WHEN ((annual_resret#94362533 = NA) OR (annual_resret#94362533 = null)) THEN null ELSE cast(annual_resret#94362533 as float) END AS annual_resret#94362555, CASE WHEN ((numcos#94362534 = NA) OR (numcos#94362534 = null)) THEN null ELSE cast(numcos#94362534 as float) END AS numcos#94362556]\n+- FileScan csv [fractile#94362530,cap#94362531,annual_ret#94362532,annual_resret#94362533,numcos#94362534] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/fr..., PartitionFilters: [], PushedFilters:... [3624032] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624033 [labelType="html" label="MapPartitionsRDD [3624033] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624034 [labelType="html" label="MapPartitionsRDD [3624034] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624035 [labelType="html" label="MapPartitionsRDD [3624035] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301858 { label="WholeStageCodegen (2)"; 3624036 [labelType="html" label="MapPartitionsRDD [3624036] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301849 { label="Union"; 3624037 [labelType="html" label="UnionRDD [3624037] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301848 { label="InMemoryTableScan"; 3624038 [labelType="html" label="MapPartitionsRDD [3624038] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624039 [labelType="html" label="Union\n:- *(1) ColumnarToRow\n: +- InMemoryTableScan [fractile#94362813, annual_ret#94362815, annual_resret#94362827, numcos#94362836]\n: +- InMemoryRelation [fractile#94362813, overall#94362814, annual_ret#94362815, annual_resret#94362827, numcos#94362836], StorageLevel(disk, memory, deserialized, 1 replicas)\n: +- *(1) Project [CASE WHEN ((fractile#94362738 = NA) OR (fractile#94362738 = null)) THEN null ELSE cast(fractile#94362738 as float) END AS fractile#94362813, CASE WHEN ((overall#94362739 = NA) OR (overall#94362739 = null)) THEN null ELSE cast(overall#94362739 as int) END AS overall#94362814, CASE WHEN ((annual_ret#94362740 = NA) OR (annual_ret#94362740 = null)) THEN null ELSE cast(annual_ret#94362740 as float) END AS annual_ret#94362815, CASE WHEN ((annual_resret#94362741 = NA) OR (annual_resret#94362741 = null)) THEN null ELSE cast(annual_resret#94362741 as float) END AS annual_resret#94362827, CASE WHEN ((numcos#94362742 = NA) OR (numcos#94362742 = null)) THEN null ELSE cast(num... [3624039] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624040 [labelType="html" label="MapPartitionsRDD [3624040] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624041 [labelType="html" label="MapPartitionsRDD [3624041] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624042 [labelType="html" label="MapPartitionsRDD [3624042] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301845 { label="WholeStageCodegen (1)"; 3624043 [labelType="html" label="MapPartitionsRDD [3624043] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301844 { label="CollectLimit"; 3624044 [labelType="html" label="MapPartitionsRDD [3624044] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624045 [labelType="html" label="MapPartitionsRDD [3624045] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624019->3624020; 3624020->3624021; 3624021->3624022; 3624022->3624023; 3624023->3624024; 3624024->3624025; 3624025->3624026; 3624026->3624027; 3624028->3624029; 3624029->3624030; 3624030->3624031; 3624031->3624032; 3624032->3624033; 3624033->3624034; 3624034->3624035; 3624035->3624036; 3624027->3624037; 3624036->3624037; 3624037->3624038; 3624038->3624039; 3624039->3624040; 3624040->3624041; 3624041->3624042; 3624042->3624043; 3624043->3624044; 3624044->3624045; }

3624023

3624032

3624039

digraph G { subgraph clusterstage_512952 { label="Stage 512952"; subgraph cluster6301844 { label="CollectLimit"; 3624046 [labelType="html" label="ShuffledRowRDD [3624046] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624047 [labelType="html" label="MapPartitionsRDD [3624047] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301868 { label="mapPartitionsInternal"; 3624048 [labelType="html" label="MapPartitionsRDD [3624048] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301869 { label="map"; 3624049 [labelType="html" label="MapPartitionsRDD [3624049] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624046->3624047; 3624047->3624048; 3624048->3624049; }

3624045,3624046

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
512952	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:21:59	8 ms	1/1			287.0 B
512951	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN ((fractile#94362738 = NA) OR (fractile#94362738 = null)) THEN null ELSE cast(fractile#94362738 as float) END AS fractile#94362813, CASE WHEN ((overall#94362739 = NA) OR (overall#94362739 = null)) THEN null ELSE cast(overall#94362739 as int) END AS overall#94362814, CASE WHEN ((annual_ret#94362740 = NA) OR (annual_ret#94362740 = null)) THEN null ELSE cast(annual_ret#94362740 as float) END AS annual_ret#94362815, CASE WHEN ((annual_resret#94362741 = NA) OR (annual_resret#94362741 = null)) THEN null ELSE cast(annual_resret#94362741 as float) END AS annual_resret#94362827, CASE WHEN ((numcos#94362742 = NA) OR (numcos#94362742 = null)) THEN null ELSE cast(numcos#94362742 as float) END AS numcos#94362836] +- FileScan csv [fractile#94362738,overall#94362739,annual_ret#94362740,annual_resret#94362741,numcos#94362742] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/fr..., PartitionFilters: ...(1) Project [CASE WHEN ((fractile#94362530 = NA) OR (fractile#94362530 = null)) THEN null ELSE cast(fractile#94362530 as float) END AS fractile#94362552, CASE WHEN ((cap#94362531 = NA) OR (cap#94362531 = null)) THEN null ELSE cast(cap#94362531 as float) END AS cap#94362553, CASE WHEN ((annual_ret#94362532 = NA) OR (annual_ret#94362532 = null)) THEN null ELSE cast(annual_ret#94362532 as float) END AS annual_ret#94362554, CASE WHEN ((annual_resret#94362533 = NA) OR (annual_resret#94362533 = null)) THEN null ELSE cast(annual_resret#94362533 as float) END AS annual_resret#94362555, CASE WHEN ((numcos#94362534 = NA) OR (numcos#94362534 = null)) THEN null ELSE cast(numcos#94362534 as float) END AS numcos#94362556] +- FileScan csv [fractile#94362530,cap#94362531,annual_ret#94362532,annual_resret#94362533,numcos#94362534] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/growth/fr..., PartitionFilters: [], PushedFilters:...Union :- (1) ColumnarToRow : +- InMemoryTableScan [fractile#94362813, annual_ret#94362815, annual_resret#94362827, numcos#94362836] : +- InMemoryRelation [fractile#94362813, overall#94362814, annual_ret#94362815, annual_resret#94362827, numcos#94362836], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN ((fractile#94362738 = NA) OR (fractile#94362738 = null)) THEN null ELSE cast(fractile#94362738 as float) END AS fractile#94362813, CASE WHEN ((overall#94362739 = NA) OR (overall#94362739 = null)) THEN null ELSE cast(overall#94362739 as int) END AS overall#94362814, CASE WHEN ((annual_ret#94362740 = NA) OR (annual_ret#94362740 = null)) THEN null ELSE cast(annual_ret#94362740 as float) END AS annual_ret#94362815, CASE WHEN ((annual_resret#94362741 = NA) OR (annual_resret#94362741 = null)) THEN null ELSE cast(annual_resret#94362741 as float) END AS annual_resret#94362827, CASE WHEN ((numcos#94362742 = NA) OR (numcos#94362742 = null)) THEN null ELSE cast(num... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:21:59	41 ms	2/2	2.7 KiB			287.0 B

Page: