PlusAMP App Middleware - Details for Job 330963

Status: SUCCEEDED
Submitted: 2025/08/02 15:06:57
Duration: 1 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512304 { label="Stage 512304"; subgraph cluster6293693 { label="Scan csv "; 3619317 [labelType="html" label="FileScanRDD [3619317] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619318 [labelType="html" label="MapPartitionsRDD [3619318] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293690 { label="WholeStageCodegen (1)"; 3619319 [labelType="html" label="MapPartitionsRDD [3619319] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293689 { label="InMemoryTableScan"; 3619320 [labelType="html" label="MapPartitionsRDD [3619320] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619321 [labelType="html" label="*(1) Project [CASE WHEN (date#94240143 = null) THEN null ELSE cast(date#94240143 as date) END AS date#94240187, CASE WHEN ((overall#94240144 = NA) OR (overall#94240144 = null)) THEN null ELSE cast(overall#94240144 as int) END AS overall#94240188, CASE WHEN ((ret#94240145 = NA) OR (ret#94240145 = null)) THEN null ELSE cast(ret#94240145 as float) END AS ret#94240201, CASE WHEN ((resret#94240146 = NA) OR (resret#94240146 = null)) THEN null ELSE cast(resret#94240146 as float) END AS resret#94240203, CASE WHEN ((retnet#94240147 = NA) OR (retnet#94240147 = null)) THEN null ELSE cast(retnet#94240147 as float) END AS retnet#94240204, CASE WHEN ((turnover#94240148 = NA) OR (turnover#94240148 = null)) THEN null ELSE cast(turnover#94240148 as float) END AS turnover#94240205, CASE WHEN ((numcos#94240149 = NA) OR (numcos#94240149 = null)) THEN null ELSE cast(numcos#94240149 as float) END AS numcos#94240206, CASE WHEN ((benchmark#94240150 = NA) OR (benchmark#94240150 = null)) THEN null ELSE cast(benchmark#94240150 as fl... [3619321] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619322 [labelType="html" label="MapPartitionsRDD [3619322] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619323 [labelType="html" label="MapPartitionsRDD [3619323] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619324 [labelType="html" label="MapPartitionsRDD [3619324] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293702 { label="Scan csv "; 3619325 [labelType="html" label="FileScanRDD [3619325] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619326 [labelType="html" label="MapPartitionsRDD [3619326] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293699 { label="WholeStageCodegen (1)"; 3619327 [labelType="html" label="MapPartitionsRDD [3619327] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293698 { label="InMemoryTableScan"; 3619328 [labelType="html" label="MapPartitionsRDD [3619328] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619329 [labelType="html" label="*(1) Project [CASE WHEN (date#94240082 = null) THEN null ELSE cast(date#94240082 as date) END AS date#94240130, CASE WHEN ((cap#94240083 = NA) OR (cap#94240083 = null)) THEN null ELSE cast(cap#94240083 as float) END AS cap#94240131, CASE WHEN ((ret#94240084 = NA) OR (ret#94240084 = null)) THEN null ELSE cast(ret#94240084 as float) END AS ret#94240132, CASE WHEN ((resret#94240085 = NA) OR (resret#94240085 = null)) THEN null ELSE cast(resret#94240085 as float) END AS resret#94240134, CASE WHEN ((retnet#94240086 = NA) OR (retnet#94240086 = null)) THEN null ELSE cast(retnet#94240086 as float) END AS retnet#94240135, CASE WHEN ((turnover#94240087 = NA) OR (turnover#94240087 = null)) THEN null ELSE cast(turnover#94240087 as float) END AS turnover#94240136, CASE WHEN ((numcos#94240088 = NA) OR (numcos#94240088 = null)) THEN null ELSE cast(numcos#94240088 as float) END AS numcos#94240137, CASE WHEN ((coverage#94240089 = NA) OR (coverage#94240089 = null)) THEN null ELSE cast(coverage#94240089 as float) END AS cover... [3619329] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619330 [labelType="html" label="MapPartitionsRDD [3619330] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619331 [labelType="html" label="MapPartitionsRDD [3619331] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619332 [labelType="html" label="MapPartitionsRDD [3619332] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293694 { label="WholeStageCodegen (1)"; 3619333 [labelType="html" label="MapPartitionsRDD [3619333] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293688 { label="Union"; 3619334 [labelType="html" label="UnionRDD [3619334] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293687 { label="Exchange"; 3619338 [labelType="html" label="MapPartitionsRDD [3619338] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3619317->3619318; 3619318->3619319; 3619319->3619320; 3619320->3619321; 3619321->3619322; 3619322->3619323; 3619323->3619324; 3619325->3619326; 3619326->3619327; 3619327->3619328; 3619328->3619329; 3619329->3619330; 3619330->3619331; 3619331->3619332; 3619332->3619333; 3619324->3619334; 3619333->3619334; 3619334->3619338; }

3619321

3619329

digraph G { subgraph clusterstage_512305 { label="Stage 512305"; subgraph cluster6293687 { label="Exchange"; 3619339 [labelType="html" label="ShuffledRowRDD [3619339] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293684 { label="WholeStageCodegen (2)"; 3619340 [labelType="html" label="MapPartitionsRDD [3619340] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293683 { label="InMemoryTableScan"; 3619341 [labelType="html" label="MapPartitionsRDD [3619341] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619342 [labelType="html" label="*(2) Sort [date#94240187 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94240187 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7524341]\n +- Union\n :- InMemoryTableScan [date#94240187, numcos#94240206]\n : +- InMemoryRelation [date#94240187, overall#94240188, ret#94240201, resret#94240203, retnet#94240204, turnover#94240205, numcos#94240206, benchmark#94240207, excess_ret#94240208, excess_resret#94240209, excess_retnet#94240210], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94240143 = null) THEN null ELSE cast(date#94240143 as date) END AS date#94240187, CASE WHEN ((overall#94240144 = NA) OR (overall#94240144 = null)) THEN null ELSE cast(overall#94240144 as int) END AS overall#94240188, CASE WHEN ((ret#94240145 = NA) OR (ret#94240145 = null)) THEN null ELSE cast(ret#94240145 as float) END AS ret#94240201, CASE WHEN ((resret#94240146 = NA) OR (resret#94240146 = null)) THEN null ELSE cast(resret#94240146 as float) END A... [3619342] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619343 [labelType="html" label="MapPartitionsRDD [3619343] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619344 [labelType="html" label="MapPartitionsRDD [3619344] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619345 [labelType="html" label="MapPartitionsRDD [3619345] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293682 { label="CollectLimit"; 3619346 [labelType="html" label="MapPartitionsRDD [3619346] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619347 [labelType="html" label="MapPartitionsRDD [3619347] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3619339->3619340; 3619340->3619341; 3619341->3619342; 3619342->3619343; 3619343->3619344; 3619344->3619345; 3619345->3619346; 3619346->3619347; }

3619338,3619339

3619342

digraph G { subgraph clusterstage_512306 { label="Stage 512306"; subgraph cluster6293682 { label="CollectLimit"; 3619348 [labelType="html" label="ShuffledRowRDD [3619348] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619349 [labelType="html" label="MapPartitionsRDD [3619349] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293724 { label="mapPartitionsInternal"; 3619350 [labelType="html" label="MapPartitionsRDD [3619350] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293725 { label="map"; 3619351 [labelType="html" label="MapPartitionsRDD [3619351] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3619348->3619349; 3619349->3619350; 3619350->3619351; }

3619347,3619348

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512306	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:06:58	19 ms	1/1		40.4 KiB
512305	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94240187 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94240187 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7524341] +- Union :- InMemoryTableScan [date#94240187, numcos#94240206] : +- InMemoryRelation [date#94240187, overall#94240188, ret#94240201, resret#94240203, retnet#94240204, turnover#94240205, numcos#94240206, benchmark#94240207, excess_ret#94240208, excess_resret#94240209, excess_retnet#94240210], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94240143 = null) THEN null ELSE cast(date#94240143 as date) END AS date#94240187, CASE WHEN ((overall#94240144 = NA) OR (overall#94240144 = null)) THEN null ELSE cast(overall#94240144 as int) END AS overall#94240188, CASE WHEN ((ret#94240145 = NA) OR (ret#94240145 = null)) THEN null ELSE cast(ret#94240145 as float) END AS ret#94240201, CASE WHEN ((resret#94240146 = NA) OR (resret#94240146 = null)) THEN null ELSE cast(resret#94240146 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:06:57	1.0 s	200/200		40.4 KiB	40.4 KiB
512304	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94240143 = null) THEN null ELSE cast(date#94240143 as date) END AS date#94240187, CASE WHEN ((overall#94240144 = NA) OR (overall#94240144 = null)) THEN null ELSE cast(overall#94240144 as int) END AS overall#94240188, CASE WHEN ((ret#94240145 = NA) OR (ret#94240145 = null)) THEN null ELSE cast(ret#94240145 as float) END AS ret#94240201, CASE WHEN ((resret#94240146 = NA) OR (resret#94240146 = null)) THEN null ELSE cast(resret#94240146 as float) END AS resret#94240203, CASE WHEN ((retnet#94240147 = NA) OR (retnet#94240147 = null)) THEN null ELSE cast(retnet#94240147 as float) END AS retnet#94240204, CASE WHEN ((turnover#94240148 = NA) OR (turnover#94240148 = null)) THEN null ELSE cast(turnover#94240148 as float) END AS turnover#94240205, CASE WHEN ((numcos#94240149 = NA) OR (numcos#94240149 = null)) THEN null ELSE cast(numcos#94240149 as float) END AS numcos#94240206, CASE WHEN ((benchmark#94240150 = NA) OR (benchmark#94240150 = null)) THEN null ELSE cast(benchmark#94240150 as fl...(1) Project [CASE WHEN (date#94240082 = null) THEN null ELSE cast(date#94240082 as date) END AS date#94240130, CASE WHEN ((cap#94240083 = NA) OR (cap#94240083 = null)) THEN null ELSE cast(cap#94240083 as float) END AS cap#94240131, CASE WHEN ((ret#94240084 = NA) OR (ret#94240084 = null)) THEN null ELSE cast(ret#94240084 as float) END AS ret#94240132, CASE WHEN ((resret#94240085 = NA) OR (resret#94240085 = null)) THEN null ELSE cast(resret#94240085 as float) END AS resret#94240134, CASE WHEN ((retnet#94240086 = NA) OR (retnet#94240086 = null)) THEN null ELSE cast(retnet#94240086 as float) END AS retnet#94240135, CASE WHEN ((turnover#94240087 = NA) OR (turnover#94240087 = null)) THEN null ELSE cast(turnover#94240087 as float) END AS turnover#94240136, CASE WHEN ((numcos#94240088 = NA) OR (numcos#94240088 = null)) THEN null ELSE cast(numcos#94240088 as float) END AS numcos#94240137, CASE WHEN ((coverage#94240089 = NA) OR (coverage#94240089 = null)) THEN null ELSE cast(coverage#94240089 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:06:57	48 ms	2/2	763.4 KiB		40.4 KiB

Page: