PlusAMP App Middleware - Details for Job 331208

Status: SUCCEEDED
Submitted: 2025/08/02 15:15:33
Duration: 1 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512674 { label="Stage 512674"; subgraph cluster6298369 { label="Scan csv "; 3622008 [labelType="html" label="FileScanRDD [3622008] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622009 [labelType="html" label="MapPartitionsRDD [3622009] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298366 { label="WholeStageCodegen (1)"; 3622010 [labelType="html" label="MapPartitionsRDD [3622010] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298365 { label="InMemoryTableScan"; 3622011 [labelType="html" label="MapPartitionsRDD [3622011] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622012 [labelType="html" label="*(1) Project [CASE WHEN (date#94310751 = null) THEN null ELSE cast(date#94310751 as date) END AS date#94310974, CASE WHEN ((overall#94310752 = NA) OR (overall#94310752 = null)) THEN null ELSE cast(overall#94310752 as int) END AS overall#94310975, CASE WHEN ((ret#94310753 = NA) OR (ret#94310753 = null)) THEN null ELSE cast(ret#94310753 as float) END AS ret#94311002, CASE WHEN ((resret#94310754 = NA) OR (resret#94310754 = null)) THEN null ELSE cast(resret#94310754 as float) END AS resret#94311003, CASE WHEN ((retnet#94310755 = NA) OR (retnet#94310755 = null)) THEN null ELSE cast(retnet#94310755 as float) END AS retnet#94311004, CASE WHEN ((turnover#94310756 = NA) OR (turnover#94310756 = null)) THEN null ELSE cast(turnover#94310756 as float) END AS turnover#94311022, CASE WHEN ((numcos#94310757 = NA) OR (numcos#94310757 = null)) THEN null ELSE cast(numcos#94310757 as float) END AS numcos#94311023, CASE WHEN ((benchmark#94310758 = NA) OR (benchmark#94310758 = null)) THEN null ELSE cast(benchmark#94310758 as fl... [3622012] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622013 [labelType="html" label="MapPartitionsRDD [3622013] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622014 [labelType="html" label="MapPartitionsRDD [3622014] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622015 [labelType="html" label="MapPartitionsRDD [3622015] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298378 { label="Scan csv "; 3622016 [labelType="html" label="FileScanRDD [3622016] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622017 [labelType="html" label="MapPartitionsRDD [3622017] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298375 { label="WholeStageCodegen (1)"; 3622018 [labelType="html" label="MapPartitionsRDD [3622018] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298374 { label="InMemoryTableScan"; 3622019 [labelType="html" label="MapPartitionsRDD [3622019] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622020 [labelType="html" label="*(1) Project [CASE WHEN (date#94310774 = null) THEN null ELSE cast(date#94310774 as date) END AS date#94311016, CASE WHEN ((cap#94310775 = NA) OR (cap#94310775 = null)) THEN null ELSE cast(cap#94310775 as float) END AS cap#94311017, CASE WHEN ((ret#94310776 = NA) OR (ret#94310776 = null)) THEN null ELSE cast(ret#94310776 as float) END AS ret#94311018, CASE WHEN ((resret#94310777 = NA) OR (resret#94310777 = null)) THEN null ELSE cast(resret#94310777 as float) END AS resret#94311025, CASE WHEN ((retnet#94310778 = NA) OR (retnet#94310778 = null)) THEN null ELSE cast(retnet#94310778 as float) END AS retnet#94311026, CASE WHEN ((turnover#94310779 = NA) OR (turnover#94310779 = null)) THEN null ELSE cast(turnover#94310779 as float) END AS turnover#94311027, CASE WHEN ((numcos#94310780 = NA) OR (numcos#94310780 = null)) THEN null ELSE cast(numcos#94310780 as float) END AS numcos#94311028, CASE WHEN ((coverage#94310781 = NA) OR (coverage#94310781 = null)) THEN null ELSE cast(coverage#94310781 as float) END AS cover... [3622020] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622021 [labelType="html" label="MapPartitionsRDD [3622021] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622022 [labelType="html" label="MapPartitionsRDD [3622022] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622023 [labelType="html" label="MapPartitionsRDD [3622023] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298370 { label="WholeStageCodegen (1)"; 3622024 [labelType="html" label="MapPartitionsRDD [3622024] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298364 { label="Union"; 3622025 [labelType="html" label="UnionRDD [3622025] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298363 { label="Exchange"; 3622029 [labelType="html" label="MapPartitionsRDD [3622029] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622008->3622009; 3622009->3622010; 3622010->3622011; 3622011->3622012; 3622012->3622013; 3622013->3622014; 3622014->3622015; 3622016->3622017; 3622017->3622018; 3622018->3622019; 3622019->3622020; 3622020->3622021; 3622021->3622022; 3622022->3622023; 3622023->3622024; 3622015->3622025; 3622024->3622025; 3622025->3622029; }

3622012

3622020

digraph G { subgraph clusterstage_512675 { label="Stage 512675"; subgraph cluster6298363 { label="Exchange"; 3622030 [labelType="html" label="ShuffledRowRDD [3622030] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298360 { label="WholeStageCodegen (2)"; 3622031 [labelType="html" label="MapPartitionsRDD [3622031] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298359 { label="InMemoryTableScan"; 3622032 [labelType="html" label="MapPartitionsRDD [3622032] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622033 [labelType="html" label="*(2) Sort [date#94310974 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94310974 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7530069]\n +- Union\n :- InMemoryTableScan [date#94310974, numcos#94311023]\n : +- InMemoryRelation [date#94310974, overall#94310975, ret#94311002, resret#94311003, retnet#94311004, turnover#94311022, numcos#94311023, benchmark#94311024, excess_ret#94311034, excess_resret#94311035, excess_retnet#94311036], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94310751 = null) THEN null ELSE cast(date#94310751 as date) END AS date#94310974, CASE WHEN ((overall#94310752 = NA) OR (overall#94310752 = null)) THEN null ELSE cast(overall#94310752 as int) END AS overall#94310975, CASE WHEN ((ret#94310753 = NA) OR (ret#94310753 = null)) THEN null ELSE cast(ret#94310753 as float) END AS ret#94311002, CASE WHEN ((resret#94310754 = NA) OR (resret#94310754 = null)) THEN null ELSE cast(resret#94310754 as float) END A... [3622033] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622034 [labelType="html" label="MapPartitionsRDD [3622034] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622035 [labelType="html" label="MapPartitionsRDD [3622035] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622036 [labelType="html" label="MapPartitionsRDD [3622036] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298358 { label="CollectLimit"; 3622037 [labelType="html" label="MapPartitionsRDD [3622037] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622038 [labelType="html" label="MapPartitionsRDD [3622038] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622030->3622031; 3622031->3622032; 3622032->3622033; 3622033->3622034; 3622034->3622035; 3622035->3622036; 3622036->3622037; 3622037->3622038; }

3622029,3622030

3622033

digraph G { subgraph clusterstage_512676 { label="Stage 512676"; subgraph cluster6298358 { label="CollectLimit"; 3622039 [labelType="html" label="ShuffledRowRDD [3622039] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622040 [labelType="html" label="MapPartitionsRDD [3622040] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298400 { label="mapPartitionsInternal"; 3622041 [labelType="html" label="MapPartitionsRDD [3622041] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6298401 { label="map"; 3622042 [labelType="html" label="MapPartitionsRDD [3622042] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622039->3622040; 3622040->3622041; 3622041->3622042; }

3622038,3622039

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512676	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:15:34	20 ms	1/1		35.4 KiB
512675	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94310974 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94310974 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7530069] +- Union :- InMemoryTableScan [date#94310974, numcos#94311023] : +- InMemoryRelation [date#94310974, overall#94310975, ret#94311002, resret#94311003, retnet#94311004, turnover#94311022, numcos#94311023, benchmark#94311024, excess_ret#94311034, excess_resret#94311035, excess_retnet#94311036], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94310751 = null) THEN null ELSE cast(date#94310751 as date) END AS date#94310974, CASE WHEN ((overall#94310752 = NA) OR (overall#94310752 = null)) THEN null ELSE cast(overall#94310752 as int) END AS overall#94310975, CASE WHEN ((ret#94310753 = NA) OR (ret#94310753 = null)) THEN null ELSE cast(ret#94310753 as float) END AS ret#94311002, CASE WHEN ((resret#94310754 = NA) OR (resret#94310754 = null)) THEN null ELSE cast(resret#94310754 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:15:33	1 s	200/200		35.4 KiB	35.4 KiB
512674	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94310751 = null) THEN null ELSE cast(date#94310751 as date) END AS date#94310974, CASE WHEN ((overall#94310752 = NA) OR (overall#94310752 = null)) THEN null ELSE cast(overall#94310752 as int) END AS overall#94310975, CASE WHEN ((ret#94310753 = NA) OR (ret#94310753 = null)) THEN null ELSE cast(ret#94310753 as float) END AS ret#94311002, CASE WHEN ((resret#94310754 = NA) OR (resret#94310754 = null)) THEN null ELSE cast(resret#94310754 as float) END AS resret#94311003, CASE WHEN ((retnet#94310755 = NA) OR (retnet#94310755 = null)) THEN null ELSE cast(retnet#94310755 as float) END AS retnet#94311004, CASE WHEN ((turnover#94310756 = NA) OR (turnover#94310756 = null)) THEN null ELSE cast(turnover#94310756 as float) END AS turnover#94311022, CASE WHEN ((numcos#94310757 = NA) OR (numcos#94310757 = null)) THEN null ELSE cast(numcos#94310757 as float) END AS numcos#94311023, CASE WHEN ((benchmark#94310758 = NA) OR (benchmark#94310758 = null)) THEN null ELSE cast(benchmark#94310758 as fl...(1) Project [CASE WHEN (date#94310774 = null) THEN null ELSE cast(date#94310774 as date) END AS date#94311016, CASE WHEN ((cap#94310775 = NA) OR (cap#94310775 = null)) THEN null ELSE cast(cap#94310775 as float) END AS cap#94311017, CASE WHEN ((ret#94310776 = NA) OR (ret#94310776 = null)) THEN null ELSE cast(ret#94310776 as float) END AS ret#94311018, CASE WHEN ((resret#94310777 = NA) OR (resret#94310777 = null)) THEN null ELSE cast(resret#94310777 as float) END AS resret#94311025, CASE WHEN ((retnet#94310778 = NA) OR (retnet#94310778 = null)) THEN null ELSE cast(retnet#94310778 as float) END AS retnet#94311026, CASE WHEN ((turnover#94310779 = NA) OR (turnover#94310779 = null)) THEN null ELSE cast(turnover#94310779 as float) END AS turnover#94311027, CASE WHEN ((numcos#94310780 = NA) OR (numcos#94310780 = null)) THEN null ELSE cast(numcos#94310780 as float) END AS numcos#94311028, CASE WHEN ((coverage#94310781 = NA) OR (coverage#94310781 = null)) THEN null ELSE cast(coverage#94310781 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:15:33	40 ms	2/2	577.9 KiB		35.4 KiB

Page: