PlusAMP App Middleware - Details for Job 331257

Status: SUCCEEDED
Submitted: 2025/08/02 15:17:16
Duration: 0.8 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512748 { label="Stage 512748"; subgraph cluster6299313 { label="Scan csv "; 3622551 [labelType="html" label="FileScanRDD [3622551] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622552 [labelType="html" label="MapPartitionsRDD [3622552] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299310 { label="WholeStageCodegen (1)"; 3622553 [labelType="html" label="MapPartitionsRDD [3622553] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299309 { label="InMemoryTableScan"; 3622554 [labelType="html" label="MapPartitionsRDD [3622554] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622555 [labelType="html" label="*(1) Project [CASE WHEN (date#94324581 = null) THEN null ELSE cast(date#94324581 as date) END AS date#94324653, CASE WHEN ((overall#94324582 = NA) OR (overall#94324582 = null)) THEN null ELSE cast(overall#94324582 as int) END AS overall#94324654, CASE WHEN ((ret#94324583 = NA) OR (ret#94324583 = null)) THEN null ELSE cast(ret#94324583 as float) END AS ret#94324655, CASE WHEN ((resret#94324584 = NA) OR (resret#94324584 = null)) THEN null ELSE cast(resret#94324584 as float) END AS resret#94324656, CASE WHEN ((retnet#94324585 = NA) OR (retnet#94324585 = null)) THEN null ELSE cast(retnet#94324585 as float) END AS retnet#94324683, CASE WHEN ((turnover#94324586 = NA) OR (turnover#94324586 = null)) THEN null ELSE cast(turnover#94324586 as float) END AS turnover#94324684, CASE WHEN ((numcos#94324587 = NA) OR (numcos#94324587 = null)) THEN null ELSE cast(numcos#94324587 as float) END AS numcos#94324711, CASE WHEN ((benchmark#94324588 = NA) OR (benchmark#94324588 = null)) THEN null ELSE cast(benchmark#94324588 as fl... [3622555] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622556 [labelType="html" label="MapPartitionsRDD [3622556] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622557 [labelType="html" label="MapPartitionsRDD [3622557] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622558 [labelType="html" label="MapPartitionsRDD [3622558] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299322 { label="Scan csv "; 3622559 [labelType="html" label="FileScanRDD [3622559] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622560 [labelType="html" label="MapPartitionsRDD [3622560] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299319 { label="WholeStageCodegen (1)"; 3622561 [labelType="html" label="MapPartitionsRDD [3622561] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299318 { label="InMemoryTableScan"; 3622562 [labelType="html" label="MapPartitionsRDD [3622562] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622563 [labelType="html" label="*(1) Project [CASE WHEN (date#94324603 = null) THEN null ELSE cast(date#94324603 as date) END AS date#94324818, CASE WHEN ((cap#94324604 = NA) OR (cap#94324604 = null)) THEN null ELSE cast(cap#94324604 as float) END AS cap#94324845, CASE WHEN ((ret#94324605 = NA) OR (ret#94324605 = null)) THEN null ELSE cast(ret#94324605 as float) END AS ret#94324872, CASE WHEN ((resret#94324606 = NA) OR (resret#94324606 = null)) THEN null ELSE cast(resret#94324606 as float) END AS resret#94324873, CASE WHEN ((retnet#94324607 = NA) OR (retnet#94324607 = null)) THEN null ELSE cast(retnet#94324607 as float) END AS retnet#94324874, CASE WHEN ((turnover#94324608 = NA) OR (turnover#94324608 = null)) THEN null ELSE cast(turnover#94324608 as float) END AS turnover#94324902, CASE WHEN ((numcos#94324609 = NA) OR (numcos#94324609 = null)) THEN null ELSE cast(numcos#94324609 as float) END AS numcos#94324915, CASE WHEN ((coverage#94324610 = NA) OR (coverage#94324610 = null)) THEN null ELSE cast(coverage#94324610 as float) END AS cover... [3622563] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622564 [labelType="html" label="MapPartitionsRDD [3622564] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622565 [labelType="html" label="MapPartitionsRDD [3622565] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622566 [labelType="html" label="MapPartitionsRDD [3622566] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299314 { label="WholeStageCodegen (1)"; 3622567 [labelType="html" label="MapPartitionsRDD [3622567] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299308 { label="Union"; 3622568 [labelType="html" label="UnionRDD [3622568] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299307 { label="Exchange"; 3622572 [labelType="html" label="MapPartitionsRDD [3622572] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622551->3622552; 3622552->3622553; 3622553->3622554; 3622554->3622555; 3622555->3622556; 3622556->3622557; 3622557->3622558; 3622559->3622560; 3622560->3622561; 3622561->3622562; 3622562->3622563; 3622563->3622564; 3622564->3622565; 3622565->3622566; 3622566->3622567; 3622558->3622568; 3622567->3622568; 3622568->3622572; }

3622555

3622563

digraph G { subgraph clusterstage_512749 { label="Stage 512749"; subgraph cluster6299307 { label="Exchange"; 3622573 [labelType="html" label="ShuffledRowRDD [3622573] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299304 { label="WholeStageCodegen (2)"; 3622574 [labelType="html" label="MapPartitionsRDD [3622574] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299303 { label="InMemoryTableScan"; 3622575 [labelType="html" label="MapPartitionsRDD [3622575] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622576 [labelType="html" label="*(2) Sort [date#94324653 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94324653 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7531219]\n +- Union\n :- InMemoryTableScan [date#94324653, numcos#94324711]\n : +- InMemoryRelation [date#94324653, overall#94324654, ret#94324655, resret#94324656, retnet#94324683, turnover#94324684, numcos#94324711, benchmark#94324712, excess_ret#94324713, excess_resret#94324752, excess_retnet#94324753], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94324581 = null) THEN null ELSE cast(date#94324581 as date) END AS date#94324653, CASE WHEN ((overall#94324582 = NA) OR (overall#94324582 = null)) THEN null ELSE cast(overall#94324582 as int) END AS overall#94324654, CASE WHEN ((ret#94324583 = NA) OR (ret#94324583 = null)) THEN null ELSE cast(ret#94324583 as float) END AS ret#94324655, CASE WHEN ((resret#94324584 = NA) OR (resret#94324584 = null)) THEN null ELSE cast(resret#94324584 as float) END A... [3622576] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622577 [labelType="html" label="MapPartitionsRDD [3622577] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622578 [labelType="html" label="MapPartitionsRDD [3622578] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622579 [labelType="html" label="MapPartitionsRDD [3622579] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299302 { label="CollectLimit"; 3622580 [labelType="html" label="MapPartitionsRDD [3622580] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622581 [labelType="html" label="MapPartitionsRDD [3622581] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622573->3622574; 3622574->3622575; 3622575->3622576; 3622576->3622577; 3622577->3622578; 3622578->3622579; 3622579->3622580; 3622580->3622581; }

3622572,3622573

3622576

digraph G { subgraph clusterstage_512750 { label="Stage 512750"; subgraph cluster6299302 { label="CollectLimit"; 3622582 [labelType="html" label="ShuffledRowRDD [3622582] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622583 [labelType="html" label="MapPartitionsRDD [3622583] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299344 { label="mapPartitionsInternal"; 3622584 [labelType="html" label="MapPartitionsRDD [3622584] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299345 { label="map"; 3622585 [labelType="html" label="MapPartitionsRDD [3622585] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622582->3622583; 3622583->3622584; 3622584->3622585; }

3622581,3622582

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512750	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:17:17	18 ms	1/1		31.3 KiB
512749	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94324653 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94324653 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7531219] +- Union :- InMemoryTableScan [date#94324653, numcos#94324711] : +- InMemoryRelation [date#94324653, overall#94324654, ret#94324655, resret#94324656, retnet#94324683, turnover#94324684, numcos#94324711, benchmark#94324712, excess_ret#94324713, excess_resret#94324752, excess_retnet#94324753], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94324581 = null) THEN null ELSE cast(date#94324581 as date) END AS date#94324653, CASE WHEN ((overall#94324582 = NA) OR (overall#94324582 = null)) THEN null ELSE cast(overall#94324582 as int) END AS overall#94324654, CASE WHEN ((ret#94324583 = NA) OR (ret#94324583 = null)) THEN null ELSE cast(ret#94324583 as float) END AS ret#94324655, CASE WHEN ((resret#94324584 = NA) OR (resret#94324584 = null)) THEN null ELSE cast(resret#94324584 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:17:16	0.7 s	200/200		31.3 KiB	31.3 KiB
512748	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94324581 = null) THEN null ELSE cast(date#94324581 as date) END AS date#94324653, CASE WHEN ((overall#94324582 = NA) OR (overall#94324582 = null)) THEN null ELSE cast(overall#94324582 as int) END AS overall#94324654, CASE WHEN ((ret#94324583 = NA) OR (ret#94324583 = null)) THEN null ELSE cast(ret#94324583 as float) END AS ret#94324655, CASE WHEN ((resret#94324584 = NA) OR (resret#94324584 = null)) THEN null ELSE cast(resret#94324584 as float) END AS resret#94324656, CASE WHEN ((retnet#94324585 = NA) OR (retnet#94324585 = null)) THEN null ELSE cast(retnet#94324585 as float) END AS retnet#94324683, CASE WHEN ((turnover#94324586 = NA) OR (turnover#94324586 = null)) THEN null ELSE cast(turnover#94324586 as float) END AS turnover#94324684, CASE WHEN ((numcos#94324587 = NA) OR (numcos#94324587 = null)) THEN null ELSE cast(numcos#94324587 as float) END AS numcos#94324711, CASE WHEN ((benchmark#94324588 = NA) OR (benchmark#94324588 = null)) THEN null ELSE cast(benchmark#94324588 as fl...(1) Project [CASE WHEN (date#94324603 = null) THEN null ELSE cast(date#94324603 as date) END AS date#94324818, CASE WHEN ((cap#94324604 = NA) OR (cap#94324604 = null)) THEN null ELSE cast(cap#94324604 as float) END AS cap#94324845, CASE WHEN ((ret#94324605 = NA) OR (ret#94324605 = null)) THEN null ELSE cast(ret#94324605 as float) END AS ret#94324872, CASE WHEN ((resret#94324606 = NA) OR (resret#94324606 = null)) THEN null ELSE cast(resret#94324606 as float) END AS resret#94324873, CASE WHEN ((retnet#94324607 = NA) OR (retnet#94324607 = null)) THEN null ELSE cast(retnet#94324607 as float) END AS retnet#94324874, CASE WHEN ((turnover#94324608 = NA) OR (turnover#94324608 = null)) THEN null ELSE cast(turnover#94324608 as float) END AS turnover#94324902, CASE WHEN ((numcos#94324609 = NA) OR (numcos#94324609 = null)) THEN null ELSE cast(numcos#94324609 as float) END AS numcos#94324915, CASE WHEN ((coverage#94324610 = NA) OR (coverage#94324610 = null)) THEN null ELSE cast(coverage#94324610 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:17:16	53 ms	2/2	426.1 KiB		31.3 KiB

Page: