PlusAMP App Middleware - Details for Job 331353

Status: SUCCEEDED
Submitted: 2025/08/02 15:20:37
Duration: 53 ms
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512892 { label="Stage 512892"; subgraph cluster6288174 { label="Scan csv "; 3616158 [labelType="html" label="FileScanRDD [3616158] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; 3616159 [labelType="html" label="MapPartitionsRDD [3616159] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6288170 { label="WholeStageCodegen (1)"; 3616160 [labelType="html" label="MapPartitionsRDD [3616160] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6288169 { label="InMemoryTableScan"; 3616161 [labelType="html" label="MapPartitionsRDD [3616161] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; 3616162 [labelType="html" label="*(1) Project [CASE WHEN ((cap#94160377 = NA) OR (cap#94160377 = null)) THEN null ELSE cast(cap#94160377 as int) END AS cap#94160394, CASE WHEN (sort#94160378 = null) THEN null ELSE sort#94160378 END AS sort#94160395, CASE WHEN (description#94160379 = null) THEN null ELSE description#94160379 END AS description#94160396, CASE WHEN ((universe#94160380 = NA) OR (universe#94160380 = null)) THEN null ELSE cast(universe#94160380 as int) END AS universe#94160397]\n+- FileScan csv [cap#94160377,sort#94160378,description#94160379,universe#94160380] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_cap.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<cap:string,sort:string,description:string,universe:string>\n [3616162] [Cached] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6301151 { label="InMemoryTableScan"; 3623610 [labelType="html" label="MapPartitionsRDD [3623610] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623611 [labelType="html" label="MapPartitionsRDD [3623611] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623612 [labelType="html" label="MapPartitionsRDD [3623612] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301129 { label="WholeStageCodegen (2)"; 3623613 [labelType="html" label="MapPartitionsRDD [3623613] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301128 { label="Exchange"; 3623617 [labelType="html" label="MapPartitionsRDD [3623617] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3616158->3616159; 3616159->3616160; 3616160->3616161; 3616161->3616162; 3616162->3623610; 3623610->3623611; 3623611->3623612; 3623612->3623613; 3623613->3623617; }

3616162

digraph G { subgraph clusterstage_512893 { label="Stage 512893"; subgraph cluster6301128 { label="Exchange"; 3623618 [labelType="html" label="ShuffledRowRDD [3623618] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301122 { label="WholeStageCodegen (3)"; 3623619 [labelType="html" label="MapPartitionsRDD [3623619] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301121 { label="InMemoryTableScan"; 3623620 [labelType="html" label="MapPartitionsRDD [3623620] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623621 [labelType="html" label="*(3) Sort [sort#94160395 ASC NULLS FIRST, description#94160396 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(sort#94160395 ASC NULLS FIRST, description#94160396 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7533447]\n +- *(2) Project [cap#94354236, numcos#94354239, numdates#94354240, sort#94160395, description#94160396, universe#94160397, (cast(numcos#94354239 as double) / cast(universe#94160397 as double)) AS coverage#94354585]\n +- *(2) BroadcastHashJoin [knownfloatingpointnormalized(normalizenanandzero(cap#94354236))], [knownfloatingpointnormalized(normalizenanandzero(cast(cap#94160394 as float)))], Inner, BuildLeft, false\n :- BroadcastExchange HashedRelationBroadcastMode(List(knownfloatingpointnormalized(normalizenanandzero(input[0, float, false]))),false), [id=#7533440]\n : +- *(1) Filter isnotnull(cap#94354236)\n : +- InMemoryTableScan [cap#94354236, numcos#94354239, numdates#94354240], [isnotnull(cap#94354236)]\n : +- InMemoryRelation [ca... [3623621] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623622 [labelType="html" label="MapPartitionsRDD [3623622] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623623 [labelType="html" label="MapPartitionsRDD [3623623] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623624 [labelType="html" label="MapPartitionsRDD [3623624] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301120 { label="CollectLimit"; 3623625 [labelType="html" label="MapPartitionsRDD [3623625] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623626 [labelType="html" label="MapPartitionsRDD [3623626] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623618->3623619; 3623619->3623620; 3623620->3623621; 3623621->3623622; 3623622->3623623; 3623623->3623624; 3623624->3623625; 3623625->3623626; }

3623617,3623618

3623621

digraph G { subgraph clusterstage_512894 { label="Stage 512894"; subgraph cluster6301120 { label="CollectLimit"; 3623627 [labelType="html" label="ShuffledRowRDD [3623627] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623628 [labelType="html" label="MapPartitionsRDD [3623628] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301160 { label="mapPartitionsInternal"; 3623629 [labelType="html" label="MapPartitionsRDD [3623629] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301161 { label="map"; 3623630 [labelType="html" label="MapPartitionsRDD [3623630] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623627->3623628; 3623628->3623629; 3623629->3623630; }

3623626,3623627

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512894	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:37	8 ms	1/1		318.0 B
512893	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (3) Sort [sort#94160395 ASC NULLS FIRST, description#94160396 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(sort#94160395 ASC NULLS FIRST, description#94160396 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7533447] +- (2) Project [cap#94354236, numcos#94354239, numdates#94354240, sort#94160395, description#94160396, universe#94160397, (cast(numcos#94354239 as double) / cast(universe#94160397 as double)) AS coverage#94354585] +- (2) BroadcastHashJoin [knownfloatingpointnormalized(normalizenanandzero(cap#94354236))], [knownfloatingpointnormalized(normalizenanandzero(cast(cap#94160394 as float)))], Inner, BuildLeft, false :- BroadcastExchange HashedRelationBroadcastMode(List(knownfloatingpointnormalized(normalizenanandzero(input[0, float, false]))),false), [id=#7533440] : +- (1) Filter isnotnull(cap#94354236) : +- InMemoryTableScan [cap#94354236, numcos#94354239, numdates#94354240], [isnotnull(cap#94354236)] : +- InMemoryRelation [ca... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:37	24 ms	3/3		318.0 B	318.0 B
512892	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((cap#94160377 = NA) OR (cap#94160377 = null)) THEN null ELSE cast(cap#94160377 as int) END AS cap#94160394, CASE WHEN (sort#94160378 = null) THEN null ELSE sort#94160378 END AS sort#94160395, CASE WHEN (description#94160379 = null) THEN null ELSE description#94160379 END AS description#94160396, CASE WHEN ((universe#94160380 = NA) OR (universe#94160380 = null)) THEN null ELSE cast(universe#94160380 as int) END AS universe#94160397] +- FileScan csv [cap#94160377,sort#94160378,description#94160379,universe#94160380] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_cap.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<cap:string,sort:string,description:string,universe:string> org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:37	9 ms	1/1	624.0 B		318.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

512894

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:20:37

8 ms

1/1

318.0 B

512893

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(3) Sort [sort#94160395 ASC NULLS FIRST, description#94160396 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(sort#94160395 ASC NULLS FIRST, description#94160396 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7533447] +- *(2) Project [cap#94354236, numcos#94354239, numdates#94354240, sort#94160395, description#94160396, universe#94160397, (cast(numcos#94354239 as double) / cast(universe#94160397 as double)) AS coverage#94354585] +- *(2) BroadcastHashJoin [knownfloatingpointnormalized(normalizenanandzero(cap#94354236))], [knownfloatingpointnormalized(normalizenanandzero(cast(cap#94160394 as float)))], Inner, BuildLeft, false :- BroadcastExchange HashedRelationBroadcastMode(List(knownfloatingpointnormalized(normalizenanandzero(input[0, float, false]))),false), [id=#7533440] : +- *(1) Filter isnotnull(cap#94354236) : +- InMemoryTableScan [cap#94354236, numcos#94354239, numdates#94354240], [isnotnull(cap#94354236)] : +- InMemoryRelation [ca...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:20:37

24 ms

3/3

318.0 B

512892

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((cap#94160377 = NA) OR (cap#94160377 = null)) THEN null ELSE cast(cap#94160377 as int) END AS cap#94160394, CASE WHEN (sort#94160378 = null) THEN null ELSE sort#94160378 END AS sort#94160395, CASE WHEN (description#94160379 = null) THEN null ELSE description#94160379 END AS description#94160396, CASE WHEN ((universe#94160380 = NA) OR (universe#94160380 = null)) THEN null ELSE cast(universe#94160380 as int) END AS universe#94160397] +- FileScan csv [cap#94160377,sort#94160378,description#94160379,universe#94160380] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_cap.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<cap:string,sort:string,description:string,universe:string>

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:20:37

9 ms

1/1

624.0 B

318.0 B

Page: