PlusAMP App Middleware - Details for Job 331448

Status: SUCCEEDED
Submitted: 2025/08/02 15:23:59
Duration: 96 ms
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_513035 { label="Stage 513035"; subgraph cluster6302966 { label="Scan csv "; 3624660 [labelType="html" label="FileScanRDD [3624660] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624661 [labelType="html" label="MapPartitionsRDD [3624661] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302951 { label="WholeStageCodegen (2)"; 3624662 [labelType="html" label="MapPartitionsRDD [3624662] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302950 { label="Exchange"; 3624666 [labelType="html" label="MapPartitionsRDD [3624666] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624660->3624661; 3624661->3624662; 3624662->3624666; }

digraph G { subgraph clusterstage_513036 { label="Stage 513036"; subgraph cluster6302950 { label="Exchange"; 3624667 [labelType="html" label="ShuffledRowRDD [3624667] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302944 { label="WholeStageCodegen (3)"; 3624668 [labelType="html" label="MapPartitionsRDD [3624668] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302943 { label="InMemoryTableScan"; 3624669 [labelType="html" label="MapPartitionsRDD [3624669] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624670 [labelType="html" label="*(3) Sort [sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535639]\n +- *(2) Project [sector_id#94382736, numcos#94382820, numdates#94382821, sort#94160419, description#94160423, universe#94383074, coverage#94382913]\n +- *(2) BroadcastHashJoin [sector_id#94382736], [sector_id#94160418], Inner, BuildRight, false\n :- *(2) Project [CASE WHEN ((sector_id#94382379 = NA) OR (sector_id#94382379 = null)) THEN null ELSE cast(sector_id#94382379 as int) END AS sector_id#94382736, CASE WHEN ((numcos#94382382 = NA) OR (numcos#94382382 = null)) THEN null ELSE cast(numcos#94382382 as float) END AS numcos#94382820, CASE WHEN ((numdates#94382383 = NA) OR (numdates#94382383 = null)) THEN null ELSE cast(numdates#94382383 as float) END AS numdates#94382821, CASE WHEN ((coverage#94382404 = NA) OR (coverage#94382404 = null)) THEN null ELSE cast(coverage#9... [3624670] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624671 [labelType="html" label="MapPartitionsRDD [3624671] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624672 [labelType="html" label="MapPartitionsRDD [3624672] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624673 [labelType="html" label="MapPartitionsRDD [3624673] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302942 { label="CollectLimit"; 3624674 [labelType="html" label="MapPartitionsRDD [3624674] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624675 [labelType="html" label="MapPartitionsRDD [3624675] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624667->3624668; 3624668->3624669; 3624669->3624670; 3624670->3624671; 3624671->3624672; 3624672->3624673; 3624673->3624674; 3624674->3624675; }

3624666,3624667

3624670

digraph G { subgraph clusterstage_513037 { label="Stage 513037"; subgraph cluster6302942 { label="CollectLimit"; 3624676 [labelType="html" label="ShuffledRowRDD [3624676] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624677 [labelType="html" label="MapPartitionsRDD [3624677] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302973 { label="mapPartitionsInternal"; 3624678 [labelType="html" label="MapPartitionsRDD [3624678] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302974 { label="map"; 3624679 [labelType="html" label="MapPartitionsRDD [3624679] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624676->3624677; 3624677->3624678; 3624678->3624679; }

3624675,3624676

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
513037	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:23:59	9 ms	1/1		1424.0 B
513036	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (3) Sort [sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535639] +- (2) Project [sector_id#94382736, numcos#94382820, numdates#94382821, sort#94160419, description#94160423, universe#94383074, coverage#94382913] +- (2) BroadcastHashJoin [sector_id#94382736], [sector_id#94160418], Inner, BuildRight, false :- (2) Project [CASE WHEN ((sector_id#94382379 = NA) OR (sector_id#94382379 = null)) THEN null ELSE cast(sector_id#94382379 as int) END AS sector_id#94382736, CASE WHEN ((numcos#94382382 = NA) OR (numcos#94382382 = null)) THEN null ELSE cast(numcos#94382382 as float) END AS numcos#94382820, CASE WHEN ((numdates#94382383 = NA) OR (numdates#94382383 = null)) THEN null ELSE cast(numdates#94382383 as float) END AS numdates#94382821, CASE WHEN ((coverage#94382404 = NA) OR (coverage#94382404 = null)) THEN null ELSE cast(coverage#9... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:23:59	64 ms	13/13		1424.0 B	1424.0 B
513035	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:23:59	13 ms	1/1	6.3 KiB		1424.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

513037

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:23:59

9 ms

1/1

1424.0 B

513036

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(3) Sort [sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535639] +- *(2) Project [sector_id#94382736, numcos#94382820, numdates#94382821, sort#94160419, description#94160423, universe#94383074, coverage#94382913] +- *(2) BroadcastHashJoin [sector_id#94382736], [sector_id#94160418], Inner, BuildRight, false :- *(2) Project [CASE WHEN ((sector_id#94382379 = NA) OR (sector_id#94382379 = null)) THEN null ELSE cast(sector_id#94382379 as int) END AS sector_id#94382736, CASE WHEN ((numcos#94382382 = NA) OR (numcos#94382382 = null)) THEN null ELSE cast(numcos#94382382 as float) END AS numcos#94382820, CASE WHEN ((numdates#94382383 = NA) OR (numdates#94382383 = null)) THEN null ELSE cast(numdates#94382383 as float) END AS numdates#94382821, CASE WHEN ((coverage#94382404 = NA) OR (coverage#94382404 = null)) THEN null ELSE cast(coverage#9...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:23:59

64 ms

13/13

1424.0 B

513035

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:23:59

13 ms

1/1

6.3 KiB

1424.0 B

Page: