PlusAMP App Middleware - Details for Job 331502

Status: SUCCEEDED
Submitted: 2025/08/02 15:25:54
Duration: 0.8 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_513118 { label="Stage 513118"; subgraph cluster6303994 { label="Scan csv "; 3625248 [labelType="html" label="FileScanRDD [3625248] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625249 [labelType="html" label="MapPartitionsRDD [3625249] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303991 { label="WholeStageCodegen (1)"; 3625250 [labelType="html" label="MapPartitionsRDD [3625250] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303990 { label="InMemoryTableScan"; 3625251 [labelType="html" label="MapPartitionsRDD [3625251] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625252 [labelType="html" label="*(1) Project [CASE WHEN (date#94396171 = null) THEN null ELSE cast(date#94396171 as date) END AS date#94396426, CASE WHEN ((overall#94396172 = NA) OR (overall#94396172 = null)) THEN null ELSE cast(overall#94396172 as int) END AS overall#94396454, CASE WHEN ((ret#94396173 = NA) OR (ret#94396173 = null)) THEN null ELSE cast(ret#94396173 as float) END AS ret#94396487, CASE WHEN ((resret#94396174 = NA) OR (resret#94396174 = null)) THEN null ELSE cast(resret#94396174 as float) END AS resret#94396488, CASE WHEN ((retnet#94396175 = NA) OR (retnet#94396175 = null)) THEN null ELSE cast(retnet#94396175 as float) END AS retnet#94396489, CASE WHEN ((turnover#94396176 = NA) OR (turnover#94396176 = null)) THEN null ELSE cast(turnover#94396176 as float) END AS turnover#94396490, CASE WHEN ((numcos#94396177 = NA) OR (numcos#94396177 = null)) THEN null ELSE cast(numcos#94396177 as float) END AS numcos#94396504, CASE WHEN ((benchmark#94396178 = NA) OR (benchmark#94396178 = null)) THEN null ELSE cast(benchmark#94396178 as fl... [3625252] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625253 [labelType="html" label="MapPartitionsRDD [3625253] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625254 [labelType="html" label="MapPartitionsRDD [3625254] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625255 [labelType="html" label="MapPartitionsRDD [3625255] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304003 { label="Scan csv "; 3625256 [labelType="html" label="FileScanRDD [3625256] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625257 [labelType="html" label="MapPartitionsRDD [3625257] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304000 { label="WholeStageCodegen (1)"; 3625258 [labelType="html" label="MapPartitionsRDD [3625258] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303999 { label="InMemoryTableScan"; 3625259 [labelType="html" label="MapPartitionsRDD [3625259] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625260 [labelType="html" label="*(1) Project [CASE WHEN (date#94396194 = null) THEN null ELSE cast(date#94396194 as date) END AS date#94396509, CASE WHEN ((cap#94396195 = NA) OR (cap#94396195 = null)) THEN null ELSE cast(cap#94396195 as float) END AS cap#94396521, CASE WHEN ((ret#94396196 = NA) OR (ret#94396196 = null)) THEN null ELSE cast(ret#94396196 as float) END AS ret#94396522, CASE WHEN ((resret#94396197 = NA) OR (resret#94396197 = null)) THEN null ELSE cast(resret#94396197 as float) END AS resret#94396525, CASE WHEN ((retnet#94396198 = NA) OR (retnet#94396198 = null)) THEN null ELSE cast(retnet#94396198 as float) END AS retnet#94396528, CASE WHEN ((turnover#94396199 = NA) OR (turnover#94396199 = null)) THEN null ELSE cast(turnover#94396199 as float) END AS turnover#94396591, CASE WHEN ((numcos#94396200 = NA) OR (numcos#94396200 = null)) THEN null ELSE cast(numcos#94396200 as float) END AS numcos#94396593, CASE WHEN ((coverage#94396201 = NA) OR (coverage#94396201 = null)) THEN null ELSE cast(coverage#94396201 as float) END AS cover... [3625260] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625261 [labelType="html" label="MapPartitionsRDD [3625261] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625262 [labelType="html" label="MapPartitionsRDD [3625262] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625263 [labelType="html" label="MapPartitionsRDD [3625263] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303995 { label="WholeStageCodegen (1)"; 3625264 [labelType="html" label="MapPartitionsRDD [3625264] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303989 { label="Union"; 3625265 [labelType="html" label="UnionRDD [3625265] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303988 { label="Exchange"; 3625269 [labelType="html" label="MapPartitionsRDD [3625269] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625248->3625249; 3625249->3625250; 3625250->3625251; 3625251->3625252; 3625252->3625253; 3625253->3625254; 3625254->3625255; 3625256->3625257; 3625257->3625258; 3625258->3625259; 3625259->3625260; 3625260->3625261; 3625261->3625262; 3625262->3625263; 3625263->3625264; 3625255->3625265; 3625264->3625265; 3625265->3625269; }

3625252

3625260

digraph G { subgraph clusterstage_513119 { label="Stage 513119"; subgraph cluster6303988 { label="Exchange"; 3625270 [labelType="html" label="ShuffledRowRDD [3625270] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303985 { label="WholeStageCodegen (2)"; 3625271 [labelType="html" label="MapPartitionsRDD [3625271] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303984 { label="InMemoryTableScan"; 3625272 [labelType="html" label="MapPartitionsRDD [3625272] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625273 [labelType="html" label="*(2) Sort [date#94396426 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94396426 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7536907]\n +- Union\n :- InMemoryTableScan [date#94396426, numcos#94396504]\n : +- InMemoryRelation [date#94396426, overall#94396454, ret#94396487, resret#94396488, retnet#94396489, turnover#94396490, numcos#94396504, benchmark#94396505, excess_ret#94396506, excess_resret#94396507, excess_retnet#94396508], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94396171 = null) THEN null ELSE cast(date#94396171 as date) END AS date#94396426, CASE WHEN ((overall#94396172 = NA) OR (overall#94396172 = null)) THEN null ELSE cast(overall#94396172 as int) END AS overall#94396454, CASE WHEN ((ret#94396173 = NA) OR (ret#94396173 = null)) THEN null ELSE cast(ret#94396173 as float) END AS ret#94396487, CASE WHEN ((resret#94396174 = NA) OR (resret#94396174 = null)) THEN null ELSE cast(resret#94396174 as float) END A... [3625273] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625274 [labelType="html" label="MapPartitionsRDD [3625274] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625275 [labelType="html" label="MapPartitionsRDD [3625275] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625276 [labelType="html" label="MapPartitionsRDD [3625276] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303983 { label="CollectLimit"; 3625277 [labelType="html" label="MapPartitionsRDD [3625277] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625278 [labelType="html" label="MapPartitionsRDD [3625278] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625270->3625271; 3625271->3625272; 3625272->3625273; 3625273->3625274; 3625274->3625275; 3625275->3625276; 3625276->3625277; 3625277->3625278; }

3625269,3625270

3625273

digraph G { subgraph clusterstage_513120 { label="Stage 513120"; subgraph cluster6303983 { label="CollectLimit"; 3625279 [labelType="html" label="ShuffledRowRDD [3625279] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625280 [labelType="html" label="MapPartitionsRDD [3625280] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304025 { label="mapPartitionsInternal"; 3625281 [labelType="html" label="MapPartitionsRDD [3625281] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304026 { label="map"; 3625282 [labelType="html" label="MapPartitionsRDD [3625282] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625279->3625280; 3625280->3625281; 3625281->3625282; }

3625278,3625279

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
513120	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:25:55	19 ms	1/1		56.4 KiB
513119	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94396426 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94396426 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7536907] +- Union :- InMemoryTableScan [date#94396426, numcos#94396504] : +- InMemoryRelation [date#94396426, overall#94396454, ret#94396487, resret#94396488, retnet#94396489, turnover#94396490, numcos#94396504, benchmark#94396505, excess_ret#94396506, excess_resret#94396507, excess_retnet#94396508], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94396171 = null) THEN null ELSE cast(date#94396171 as date) END AS date#94396426, CASE WHEN ((overall#94396172 = NA) OR (overall#94396172 = null)) THEN null ELSE cast(overall#94396172 as int) END AS overall#94396454, CASE WHEN ((ret#94396173 = NA) OR (ret#94396173 = null)) THEN null ELSE cast(ret#94396173 as float) END AS ret#94396487, CASE WHEN ((resret#94396174 = NA) OR (resret#94396174 = null)) THEN null ELSE cast(resret#94396174 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:25:54	0.7 s	200/200		56.4 KiB	56.4 KiB
513118	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94396171 = null) THEN null ELSE cast(date#94396171 as date) END AS date#94396426, CASE WHEN ((overall#94396172 = NA) OR (overall#94396172 = null)) THEN null ELSE cast(overall#94396172 as int) END AS overall#94396454, CASE WHEN ((ret#94396173 = NA) OR (ret#94396173 = null)) THEN null ELSE cast(ret#94396173 as float) END AS ret#94396487, CASE WHEN ((resret#94396174 = NA) OR (resret#94396174 = null)) THEN null ELSE cast(resret#94396174 as float) END AS resret#94396488, CASE WHEN ((retnet#94396175 = NA) OR (retnet#94396175 = null)) THEN null ELSE cast(retnet#94396175 as float) END AS retnet#94396489, CASE WHEN ((turnover#94396176 = NA) OR (turnover#94396176 = null)) THEN null ELSE cast(turnover#94396176 as float) END AS turnover#94396490, CASE WHEN ((numcos#94396177 = NA) OR (numcos#94396177 = null)) THEN null ELSE cast(numcos#94396177 as float) END AS numcos#94396504, CASE WHEN ((benchmark#94396178 = NA) OR (benchmark#94396178 = null)) THEN null ELSE cast(benchmark#94396178 as fl...(1) Project [CASE WHEN (date#94396194 = null) THEN null ELSE cast(date#94396194 as date) END AS date#94396509, CASE WHEN ((cap#94396195 = NA) OR (cap#94396195 = null)) THEN null ELSE cast(cap#94396195 as float) END AS cap#94396521, CASE WHEN ((ret#94396196 = NA) OR (ret#94396196 = null)) THEN null ELSE cast(ret#94396196 as float) END AS ret#94396522, CASE WHEN ((resret#94396197 = NA) OR (resret#94396197 = null)) THEN null ELSE cast(resret#94396197 as float) END AS resret#94396525, CASE WHEN ((retnet#94396198 = NA) OR (retnet#94396198 = null)) THEN null ELSE cast(retnet#94396198 as float) END AS retnet#94396528, CASE WHEN ((turnover#94396199 = NA) OR (turnover#94396199 = null)) THEN null ELSE cast(turnover#94396199 as float) END AS turnover#94396591, CASE WHEN ((numcos#94396200 = NA) OR (numcos#94396200 = null)) THEN null ELSE cast(numcos#94396200 as float) END AS numcos#94396593, CASE WHEN ((coverage#94396201 = NA) OR (coverage#94396201 = null)) THEN null ELSE cast(coverage#94396201 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:25:54	53 ms	2/2	1084.8 KiB		56.4 KiB

Page: