PlusAMP App Middleware - Details for Job 331301

Status: SUCCEEDED
Submitted: 2025/08/02 15:18:49
Duration: 0.2 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512813 { label="Stage 512813"; subgraph cluster6288107 { label="Scan csv "; 3616122 [labelType="html" label="FileScanRDD [3616122] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; 3616123 [labelType="html" label="MapPartitionsRDD [3616123] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6288103 { label="WholeStageCodegen (1)"; 3616124 [labelType="html" label="MapPartitionsRDD [3616124] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6288102 { label="InMemoryTableScan"; 3616125 [labelType="html" label="MapPartitionsRDD [3616125] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; 3616126 [labelType="html" label="*(1) Project [CASE WHEN ((sector_id#94160398 = NA) OR (sector_id#94160398 = null)) THEN null ELSE cast(sector_id#94160398 as int) END AS sector_id#94160418, CASE WHEN (sort#94160399 = null) THEN null ELSE sort#94160399 END AS sort#94160419, CASE WHEN (description#94160400 = null) THEN null ELSE description#94160400 END AS description#94160423, CASE WHEN ((universe#94160401 = NA) OR (universe#94160401 = null)) THEN null ELSE cast(universe#94160401 as int) END AS universe#94160424]\n+- FileScan csv [sector_id#94160398,sort#94160399,description#94160400,universe#94160401] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_sector.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<sector_id:string,sort:string,description:string,universe:string>\n [3616126] [Cached] $anonfun$withThreadLocalCaptured$1 at FutureTask.java:264"]; } subgraph cluster6300160 { label="InMemoryTableScan"; 3623042 [labelType="html" label="MapPartitionsRDD [3623042] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623043 [labelType="html" label="MapPartitionsRDD [3623043] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623044 [labelType="html" label="MapPartitionsRDD [3623044] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300137 { label="WholeStageCodegen (2)"; 3623045 [labelType="html" label="MapPartitionsRDD [3623045] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300136 { label="Exchange"; 3623049 [labelType="html" label="MapPartitionsRDD [3623049] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3616122->3616123; 3616123->3616124; 3616124->3616125; 3616125->3616126; 3616126->3623042; 3623042->3623043; 3623043->3623044; 3623044->3623045; 3623045->3623049; }

3616126

digraph G { subgraph clusterstage_512814 { label="Stage 512814"; subgraph cluster6300136 { label="Exchange"; 3623050 [labelType="html" label="ShuffledRowRDD [3623050] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300129 { label="WholeStageCodegen (3)"; 3623051 [labelType="html" label="MapPartitionsRDD [3623051] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300128 { label="InMemoryTableScan"; 3623052 [labelType="html" label="MapPartitionsRDD [3623052] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623053 [labelType="html" label="*(3) Sort [sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532240]\n +- *(2) Project [sector_id#94339267, numcos#94339270, numdates#94339271, sort#94160419, description#94160423, universe#94339595, coverage#94339521]\n +- *(2) BroadcastHashJoin [sector_id#94339267], [sector_id#94160418], Inner, BuildLeft, false\n :- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7532233]\n : +- *(1) Project [sector_id#94339267, numcos#94339270, numdates#94339271, coverage#94339521, round((cast(numcos#94339270 as double) / cast(coverage#94339521 as double)), 0) AS universe#94339595]\n : +- *(1) Filter isnotnull(sector_id#94339267)\n : +- *(1) ColumnarToRow\n : +- InMemoryTableScan [coverage#94339521, numcos#94339270, numdates#94339271, sector_id#94339... [3623053] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623054 [labelType="html" label="MapPartitionsRDD [3623054] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623055 [labelType="html" label="MapPartitionsRDD [3623055] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623056 [labelType="html" label="MapPartitionsRDD [3623056] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300127 { label="CollectLimit"; 3623057 [labelType="html" label="MapPartitionsRDD [3623057] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623058 [labelType="html" label="MapPartitionsRDD [3623058] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623050->3623051; 3623051->3623052; 3623052->3623053; 3623053->3623054; 3623054->3623055; 3623055->3623056; 3623056->3623057; 3623057->3623058; }

3623049,3623050

3623053

digraph G { subgraph clusterstage_512815 { label="Stage 512815"; subgraph cluster6300127 { label="CollectLimit"; 3623059 [labelType="html" label="ShuffledRowRDD [3623059] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623060 [labelType="html" label="MapPartitionsRDD [3623060] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300169 { label="mapPartitionsInternal"; 3623061 [labelType="html" label="MapPartitionsRDD [3623061] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300170 { label="map"; 3623062 [labelType="html" label="MapPartitionsRDD [3623062] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623059->3623060; 3623060->3623061; 3623061->3623062; }

3623058,3623059

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512815	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:18:49	10 ms	1/1		1331.0 B
512814	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (3) Sort [sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532240] +- (2) Project [sector_id#94339267, numcos#94339270, numdates#94339271, sort#94160419, description#94160423, universe#94339595, coverage#94339521] +- (2) BroadcastHashJoin [sector_id#94339267], [sector_id#94160418], Inner, BuildLeft, false :- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7532233] : +- (1) Project [sector_id#94339267, numcos#94339270, numdates#94339271, coverage#94339521, round((cast(numcos#94339270 as double) / cast(coverage#94339521 as double)), 0) AS universe#94339595] : +- (1) Filter isnotnull(sector_id#94339267) : +- (1) ColumnarToRow : +- InMemoryTableScan [coverage#94339521, numcos#94339270, numdates#94339271, sector_id#94339... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:18:49	99 ms	13/13		1331.0 B	1331.0 B
512813	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((sector_id#94160398 = NA) OR (sector_id#94160398 = null)) THEN null ELSE cast(sector_id#94160398 as int) END AS sector_id#94160418, CASE WHEN (sort#94160399 = null) THEN null ELSE sort#94160399 END AS sort#94160419, CASE WHEN (description#94160400 = null) THEN null ELSE description#94160400 END AS description#94160423, CASE WHEN ((universe#94160401 = NA) OR (universe#94160401 = null)) THEN null ELSE cast(universe#94160401 as int) END AS universe#94160424] +- FileScan csv [sector_id#94160398,sort#94160399,description#94160400,universe#94160401] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_sector.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<sector_id:string,sort:string,description:string,universe:string> org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:18:49	15 ms	1/1	896.0 B		1331.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

512815

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:18:49

10 ms

1/1

1331.0 B

512814

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(3) Sort [sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(sort#94160419 ASC NULLS FIRST, description#94160423 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532240] +- *(2) Project [sector_id#94339267, numcos#94339270, numdates#94339271, sort#94160419, description#94160423, universe#94339595, coverage#94339521] +- *(2) BroadcastHashJoin [sector_id#94339267], [sector_id#94160418], Inner, BuildLeft, false :- BroadcastExchange HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [id=#7532233] : +- *(1) Project [sector_id#94339267, numcos#94339270, numdates#94339271, coverage#94339521, round((cast(numcos#94339270 as double) / cast(coverage#94339521 as double)), 0) AS universe#94339595] : +- *(1) Filter isnotnull(sector_id#94339267) : +- *(1) ColumnarToRow : +- InMemoryTableScan [coverage#94339521, numcos#94339270, numdates#94339271, sector_id#94339...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:18:49

99 ms

13/13

1331.0 B

512813

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((sector_id#94160398 = NA) OR (sector_id#94160398 = null)) THEN null ELSE cast(sector_id#94160398 as int) END AS sector_id#94160418, CASE WHEN (sort#94160399 = null) THEN null ELSE sort#94160399 END AS sort#94160419, CASE WHEN (description#94160400 = null) THEN null ELSE description#94160400 END AS description#94160423, CASE WHEN ((universe#94160401 = NA) OR (universe#94160401 = null)) THEN null ELSE cast(universe#94160401 as int) END AS universe#94160424] +- FileScan csv [sector_id#94160398,sort#94160399,description#94160400,universe#94160401] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/curate/curate_sector.csv], PartitionFilters: [], PushedFilters: [], ReadSchema: struct<sector_id:string,sort:string,description:string,universe:string>

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
java.base/java.lang.Thread.run(Thread.java:829)

2025/08/02 15:18:49

15 ms

1/1

896.0 B

1331.0 B

Page: