PlusAMP App Middleware - Details for Job 331159

Status: SUCCEEDED
Submitted: 2025/08/02 15:13:49
Duration: 0.8 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512600 { label="Stage 512600"; subgraph cluster6297436 { label="Scan csv "; 3621471 [labelType="html" label="FileScanRDD [3621471] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621472 [labelType="html" label="MapPartitionsRDD [3621472] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297433 { label="WholeStageCodegen (1)"; 3621473 [labelType="html" label="MapPartitionsRDD [3621473] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297432 { label="InMemoryTableScan"; 3621474 [labelType="html" label="MapPartitionsRDD [3621474] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621475 [labelType="html" label="*(1) Project [CASE WHEN (date#94296240 = null) THEN null ELSE cast(date#94296240 as date) END AS date#94296415, CASE WHEN ((overall#94296241 = NA) OR (overall#94296241 = null)) THEN null ELSE cast(overall#94296241 as int) END AS overall#94296416, CASE WHEN ((ret#94296242 = NA) OR (ret#94296242 = null)) THEN null ELSE cast(ret#94296242 as float) END AS ret#94296443, CASE WHEN ((resret#94296243 = NA) OR (resret#94296243 = null)) THEN null ELSE cast(resret#94296243 as float) END AS resret#94296444, CASE WHEN ((retnet#94296244 = NA) OR (retnet#94296244 = null)) THEN null ELSE cast(retnet#94296244 as float) END AS retnet#94296445, CASE WHEN ((turnover#94296245 = NA) OR (turnover#94296245 = null)) THEN null ELSE cast(turnover#94296245 as float) END AS turnover#94296472, CASE WHEN ((numcos#94296246 = NA) OR (numcos#94296246 = null)) THEN null ELSE cast(numcos#94296246 as float) END AS numcos#94296473, CASE WHEN ((benchmark#94296247 = NA) OR (benchmark#94296247 = null)) THEN null ELSE cast(benchmark#94296247 as fl... [3621475] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621476 [labelType="html" label="MapPartitionsRDD [3621476] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621477 [labelType="html" label="MapPartitionsRDD [3621477] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621478 [labelType="html" label="MapPartitionsRDD [3621478] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297445 { label="Scan csv "; 3621479 [labelType="html" label="FileScanRDD [3621479] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621480 [labelType="html" label="MapPartitionsRDD [3621480] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297442 { label="WholeStageCodegen (1)"; 3621481 [labelType="html" label="MapPartitionsRDD [3621481] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297441 { label="InMemoryTableScan"; 3621482 [labelType="html" label="MapPartitionsRDD [3621482] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621483 [labelType="html" label="*(1) Project [CASE WHEN (date#94296317 = null) THEN null ELSE cast(date#94296317 as date) END AS date#94296504, CASE WHEN ((cap#94296318 = NA) OR (cap#94296318 = null)) THEN null ELSE cast(cap#94296318 as float) END AS cap#94296521, CASE WHEN ((ret#94296319 = NA) OR (ret#94296319 = null)) THEN null ELSE cast(ret#94296319 as float) END AS ret#94296523, CASE WHEN ((resret#94296320 = NA) OR (resret#94296320 = null)) THEN null ELSE cast(resret#94296320 as float) END AS resret#94296525, CASE WHEN ((retnet#94296321 = NA) OR (retnet#94296321 = null)) THEN null ELSE cast(retnet#94296321 as float) END AS retnet#94296527, CASE WHEN ((turnover#94296322 = NA) OR (turnover#94296322 = null)) THEN null ELSE cast(turnover#94296322 as float) END AS turnover#94296529, CASE WHEN ((numcos#94296323 = NA) OR (numcos#94296323 = null)) THEN null ELSE cast(numcos#94296323 as float) END AS numcos#94296531, CASE WHEN ((coverage#94296324 = NA) OR (coverage#94296324 = null)) THEN null ELSE cast(coverage#94296324 as float) END AS cover... [3621483] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621484 [labelType="html" label="MapPartitionsRDD [3621484] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621485 [labelType="html" label="MapPartitionsRDD [3621485] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621486 [labelType="html" label="MapPartitionsRDD [3621486] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297437 { label="WholeStageCodegen (1)"; 3621487 [labelType="html" label="MapPartitionsRDD [3621487] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297431 { label="Union"; 3621488 [labelType="html" label="UnionRDD [3621488] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297430 { label="Exchange"; 3621492 [labelType="html" label="MapPartitionsRDD [3621492] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3621471->3621472; 3621472->3621473; 3621473->3621474; 3621474->3621475; 3621475->3621476; 3621476->3621477; 3621477->3621478; 3621479->3621480; 3621480->3621481; 3621481->3621482; 3621482->3621483; 3621483->3621484; 3621484->3621485; 3621485->3621486; 3621486->3621487; 3621478->3621488; 3621487->3621488; 3621488->3621492; }

3621475

3621483

digraph G { subgraph clusterstage_512601 { label="Stage 512601"; subgraph cluster6297430 { label="Exchange"; 3621493 [labelType="html" label="ShuffledRowRDD [3621493] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297427 { label="WholeStageCodegen (2)"; 3621494 [labelType="html" label="MapPartitionsRDD [3621494] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297426 { label="InMemoryTableScan"; 3621495 [labelType="html" label="MapPartitionsRDD [3621495] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621496 [labelType="html" label="*(2) Sort [date#94296415 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94296415 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7528929]\n +- Union\n :- InMemoryTableScan [date#94296415, numcos#94296473]\n : +- InMemoryRelation [date#94296415, overall#94296416, ret#94296443, resret#94296444, retnet#94296445, turnover#94296472, numcos#94296473, benchmark#94296474, excess_ret#94296475, excess_resret#94296476, excess_retnet#94296477], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94296240 = null) THEN null ELSE cast(date#94296240 as date) END AS date#94296415, CASE WHEN ((overall#94296241 = NA) OR (overall#94296241 = null)) THEN null ELSE cast(overall#94296241 as int) END AS overall#94296416, CASE WHEN ((ret#94296242 = NA) OR (ret#94296242 = null)) THEN null ELSE cast(ret#94296242 as float) END AS ret#94296443, CASE WHEN ((resret#94296243 = NA) OR (resret#94296243 = null)) THEN null ELSE cast(resret#94296243 as float) END A... [3621496] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621497 [labelType="html" label="MapPartitionsRDD [3621497] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621498 [labelType="html" label="MapPartitionsRDD [3621498] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621499 [labelType="html" label="MapPartitionsRDD [3621499] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297425 { label="CollectLimit"; 3621500 [labelType="html" label="MapPartitionsRDD [3621500] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621501 [labelType="html" label="MapPartitionsRDD [3621501] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3621493->3621494; 3621494->3621495; 3621495->3621496; 3621496->3621497; 3621497->3621498; 3621498->3621499; 3621499->3621500; 3621500->3621501; }

3621492,3621493

3621496

digraph G { subgraph clusterstage_512602 { label="Stage 512602"; subgraph cluster6297425 { label="CollectLimit"; 3621502 [labelType="html" label="ShuffledRowRDD [3621502] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3621503 [labelType="html" label="MapPartitionsRDD [3621503] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297467 { label="mapPartitionsInternal"; 3621504 [labelType="html" label="MapPartitionsRDD [3621504] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6297468 { label="map"; 3621505 [labelType="html" label="MapPartitionsRDD [3621505] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3621502->3621503; 3621503->3621504; 3621504->3621505; }

3621501,3621502

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512602	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:13:49	20 ms	1/1		55.5 KiB
512601	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94296415 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94296415 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7528929] +- Union :- InMemoryTableScan [date#94296415, numcos#94296473] : +- InMemoryRelation [date#94296415, overall#94296416, ret#94296443, resret#94296444, retnet#94296445, turnover#94296472, numcos#94296473, benchmark#94296474, excess_ret#94296475, excess_resret#94296476, excess_retnet#94296477], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94296240 = null) THEN null ELSE cast(date#94296240 as date) END AS date#94296415, CASE WHEN ((overall#94296241 = NA) OR (overall#94296241 = null)) THEN null ELSE cast(overall#94296241 as int) END AS overall#94296416, CASE WHEN ((ret#94296242 = NA) OR (ret#94296242 = null)) THEN null ELSE cast(ret#94296242 as float) END AS ret#94296443, CASE WHEN ((resret#94296243 = NA) OR (resret#94296243 = null)) THEN null ELSE cast(resret#94296243 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:13:49	0.8 s	200/200		55.5 KiB	55.5 KiB
512600	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94296240 = null) THEN null ELSE cast(date#94296240 as date) END AS date#94296415, CASE WHEN ((overall#94296241 = NA) OR (overall#94296241 = null)) THEN null ELSE cast(overall#94296241 as int) END AS overall#94296416, CASE WHEN ((ret#94296242 = NA) OR (ret#94296242 = null)) THEN null ELSE cast(ret#94296242 as float) END AS ret#94296443, CASE WHEN ((resret#94296243 = NA) OR (resret#94296243 = null)) THEN null ELSE cast(resret#94296243 as float) END AS resret#94296444, CASE WHEN ((retnet#94296244 = NA) OR (retnet#94296244 = null)) THEN null ELSE cast(retnet#94296244 as float) END AS retnet#94296445, CASE WHEN ((turnover#94296245 = NA) OR (turnover#94296245 = null)) THEN null ELSE cast(turnover#94296245 as float) END AS turnover#94296472, CASE WHEN ((numcos#94296246 = NA) OR (numcos#94296246 = null)) THEN null ELSE cast(numcos#94296246 as float) END AS numcos#94296473, CASE WHEN ((benchmark#94296247 = NA) OR (benchmark#94296247 = null)) THEN null ELSE cast(benchmark#94296247 as fl...(1) Project [CASE WHEN (date#94296317 = null) THEN null ELSE cast(date#94296317 as date) END AS date#94296504, CASE WHEN ((cap#94296318 = NA) OR (cap#94296318 = null)) THEN null ELSE cast(cap#94296318 as float) END AS cap#94296521, CASE WHEN ((ret#94296319 = NA) OR (ret#94296319 = null)) THEN null ELSE cast(ret#94296319 as float) END AS ret#94296523, CASE WHEN ((resret#94296320 = NA) OR (resret#94296320 = null)) THEN null ELSE cast(resret#94296320 as float) END AS resret#94296525, CASE WHEN ((retnet#94296321 = NA) OR (retnet#94296321 = null)) THEN null ELSE cast(retnet#94296321 as float) END AS retnet#94296527, CASE WHEN ((turnover#94296322 = NA) OR (turnover#94296322 = null)) THEN null ELSE cast(turnover#94296322 as float) END AS turnover#94296529, CASE WHEN ((numcos#94296323 = NA) OR (numcos#94296323 = null)) THEN null ELSE cast(numcos#94296323 as float) END AS numcos#94296531, CASE WHEN ((coverage#94296324 = NA) OR (coverage#94296324 = null)) THEN null ELSE cast(coverage#94296324 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:13:49	43 ms	2/2	953.1 KiB		55.5 KiB

Page: