PlusAMP App Middleware - Details for Job 331316

Status: SUCCEEDED
Submitted: 2025/08/02 15:19:05
Duration: 95 ms
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512836 { label="Stage 512836"; subgraph cluster6288174 { label="Scan csv "; 3616158 [labelType="html" label="FileScanRDD [3616158] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; 3616159 [labelType="html" label="MapPartitionsRDD [3616159] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6288170 { label="WholeStageCodegen (1)"; 3616160 [labelType="html" label="MapPartitionsRDD [3616160] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6288169 { label="InMemoryTableScan"; 3616161 [labelType="html" label="MapPartitionsRDD [3616161] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; 3616162 [labelType="html" label="*(1) Project [CASE WHEN ((cap#94160377 = NA) OR (cap#94160377 = null)) THEN null ELSE cast(cap#94160377 as int) END AS cap#94160394, CASE WHEN (sort#94160378 = null) THEN null ELSE sort#94160378 END AS sort#94160395, CASE WHEN (description#94160379 = null) THEN null ELSE description#94160379 END AS description#94160396, CASE WHEN ((universe#94160380 = NA) OR (universe#94160380 = null)) THEN null ELSE cast(universe#94160380 as int) END AS universe#94160397]\n+- FileScan csv [cap#94160377,sort#94160378,description#94160379,universe#94160380] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_cap.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<cap:string,sort:string,description:string,universe:string>\n [3616162] [Cached] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6300416 { label="InMemoryTableScan"; 3623192 [labelType="html" label="MapPartitionsRDD [3623192] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623194 [labelType="html" label="MapPartitionsRDD [3623194] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623197 [labelType="html" label="MapPartitionsRDD [3623197] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300373 { label="WholeStageCodegen (2)"; 3623199 [labelType="html" label="MapPartitionsRDD [3623199] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300372 { label="Exchange"; 3623218 [labelType="html" label="MapPartitionsRDD [3623218] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3616158->3616159; 3616159->3616160; 3616160->3616161; 3616161->3616162; 3616162->3623192; 3623192->3623194; 3623194->3623197; 3623197->3623199; 3623199->3623218; }

3616162

digraph G { subgraph clusterstage_512837 { label="Stage 512837"; subgraph cluster6300372 { label="Exchange"; 3623219 [labelType="html" label="ShuffledRowRDD [3623219] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300362 { label="WholeStageCodegen (3)"; 3623220 [labelType="html" label="MapPartitionsRDD [3623220] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300361 { label="InMemoryTableScan"; 3623221 [labelType="html" label="MapPartitionsRDD [3623221] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623222 [labelType="html" label="*(3) Project [cap#94342537, turnover#94339491, (1.0 / cast(turnover#94339491 as double)) AS days_hold#94342568]\n+- *(3) Sort [cap_sort#94342448 ASC NULLS FIRST], true, 0\n +- Exchange rangepartitioning(cap_sort#94342448 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532552]\n +- *(2) Project [turnover#94339491, cap_description#94342447 AS cap#94342537, cap_sort#94342448]\n +- *(2) BroadcastHashJoin [knownfloatingpointnormalized(normalizenanandzero(cap#94339344))], [knownfloatingpointnormalized(normalizenanandzero(cast(cap#94160394 as float)))], Inner, BuildLeft, false\n :- BroadcastExchange HashedRelationBroadcastMode(List(knownfloatingpointnormalized(normalizenanandzero(input[0, float, false]))),false), [id=#7532544]\n : +- *(1) Filter isnotnull(cap#94339344)\n : +- *(1) ColumnarToRow\n : +- InMemoryTableScan [cap#94339344, turnover#94339491], [isnotnull(cap#94339344)]\n : +- InMemoryRelation [cap#94339344, retIC#943... [3623222] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623223 [labelType="html" label="MapPartitionsRDD [3623223] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623224 [labelType="html" label="MapPartitionsRDD [3623224] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623225 [labelType="html" label="MapPartitionsRDD [3623225] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300360 { label="CollectLimit"; 3623226 [labelType="html" label="MapPartitionsRDD [3623226] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623227 [labelType="html" label="MapPartitionsRDD [3623227] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623219->3623220; 3623220->3623221; 3623221->3623222; 3623222->3623223; 3623223->3623224; 3623224->3623225; 3623225->3623226; 3623226->3623227; }

3623218,3623219

3623222

digraph G { subgraph clusterstage_512838 { label="Stage 512838"; subgraph cluster6300360 { label="CollectLimit"; 3623228 [labelType="html" label="ShuffledRowRDD [3623228] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623229 [labelType="html" label="MapPartitionsRDD [3623229] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300469 { label="mapPartitionsInternal"; 3623230 [labelType="html" label="MapPartitionsRDD [3623230] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300470 { label="map"; 3623231 [labelType="html" label="MapPartitionsRDD [3623231] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623228->3623229; 3623229->3623230; 3623230->3623231; }

3623227,3623228

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512838	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:19:06	11 ms	1/1		154.0 B
512837	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (3) Project [cap#94342537, turnover#94339491, (1.0 / cast(turnover#94339491 as double)) AS days_hold#94342568] +- (3) Sort [cap_sort#94342448 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(cap_sort#94342448 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532552] +- (2) Project [turnover#94339491, cap_description#94342447 AS cap#94342537, cap_sort#94342448] +- (2) BroadcastHashJoin [knownfloatingpointnormalized(normalizenanandzero(cap#94339344))], [knownfloatingpointnormalized(normalizenanandzero(cast(cap#94160394 as float)))], Inner, BuildLeft, false :- BroadcastExchange HashedRelationBroadcastMode(List(knownfloatingpointnormalized(normalizenanandzero(input[0, float, false]))),false), [id=#7532544] : +- (1) Filter isnotnull(cap#94339344) : +- (1) ColumnarToRow : +- InMemoryTableScan [cap#94339344, turnover#94339491], [isnotnull(cap#94339344)] : +- InMemoryRelation [cap#94339344, retIC#943... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:19:05	54 ms	2/2		134.0 B	154.0 B
512836	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((cap#94160377 = NA) OR (cap#94160377 = null)) THEN null ELSE cast(cap#94160377 as int) END AS cap#94160394, CASE WHEN (sort#94160378 = null) THEN null ELSE sort#94160378 END AS sort#94160395, CASE WHEN (description#94160379 = null) THEN null ELSE description#94160379 END AS description#94160396, CASE WHEN ((universe#94160380 = NA) OR (universe#94160380 = null)) THEN null ELSE cast(universe#94160380 as int) END AS universe#94160397] +- FileScan csv [cap#94160377,sort#94160378,description#94160379,universe#94160380] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_cap.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<cap:string,sort:string,description:string,universe:string> org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:19:05	15 ms	1/1	624.0 B		134.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

512838

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:19:06

11 ms

1/1

154.0 B

512837

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(3) Project [cap#94342537, turnover#94339491, (1.0 / cast(turnover#94339491 as double)) AS days_hold#94342568] +- *(3) Sort [cap_sort#94342448 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(cap_sort#94342448 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532552] +- *(2) Project [turnover#94339491, cap_description#94342447 AS cap#94342537, cap_sort#94342448] +- *(2) BroadcastHashJoin [knownfloatingpointnormalized(normalizenanandzero(cap#94339344))], [knownfloatingpointnormalized(normalizenanandzero(cast(cap#94160394 as float)))], Inner, BuildLeft, false :- BroadcastExchange HashedRelationBroadcastMode(List(knownfloatingpointnormalized(normalizenanandzero(input[0, float, false]))),false), [id=#7532544] : +- *(1) Filter isnotnull(cap#94339344) : +- *(1) ColumnarToRow : +- InMemoryTableScan [cap#94339344, turnover#94339491], [isnotnull(cap#94339344)] : +- InMemoryRelation [cap#94339344, retIC#943...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:19:05

54 ms

2/2

134.0 B

154.0 B

512836

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((cap#94160377 = NA) OR (cap#94160377 = null)) THEN null ELSE cast(cap#94160377 as int) END AS cap#94160394, CASE WHEN (sort#94160378 = null) THEN null ELSE sort#94160378 END AS sort#94160395, CASE WHEN (description#94160379 = null) THEN null ELSE description#94160379 END AS description#94160396, CASE WHEN ((universe#94160380 = NA) OR (universe#94160380 = null)) THEN null ELSE cast(universe#94160380 as int) END AS universe#94160397] +- FileScan csv [cap#94160377,sort#94160378,description#94160379,universe#94160380] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_cap.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<cap:string,sort:string,description:string,universe:string>

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:19:05

15 ms

1/1

624.0 B

134.0 B

Page: