PlusAMP App Middleware - Details for Job 331404

Status: SUCCEEDED
Submitted: 2025/08/02 15:22:26
Duration: 0.8 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512970 { label="Stage 512970"; subgraph cluster6302128 { label="Scan csv "; 3624174 [labelType="html" label="FileScanRDD [3624174] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624175 [labelType="html" label="MapPartitionsRDD [3624175] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302125 { label="WholeStageCodegen (1)"; 3624176 [labelType="html" label="MapPartitionsRDD [3624176] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302124 { label="InMemoryTableScan"; 3624177 [labelType="html" label="MapPartitionsRDD [3624177] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624178 [labelType="html" label="*(1) Project [CASE WHEN (date#94367711 = null) THEN null ELSE cast(date#94367711 as date) END AS date#94367886, CASE WHEN ((overall#94367712 = NA) OR (overall#94367712 = null)) THEN null ELSE cast(overall#94367712 as int) END AS overall#94367887, CASE WHEN ((ret#94367713 = NA) OR (ret#94367713 = null)) THEN null ELSE cast(ret#94367713 as float) END AS ret#94367888, CASE WHEN ((resret#94367714 = NA) OR (resret#94367714 = null)) THEN null ELSE cast(resret#94367714 as float) END AS resret#94367889, CASE WHEN ((retnet#94367715 = NA) OR (retnet#94367715 = null)) THEN null ELSE cast(retnet#94367715 as float) END AS retnet#94367890, CASE WHEN ((turnover#94367716 = NA) OR (turnover#94367716 = null)) THEN null ELSE cast(turnover#94367716 as float) END AS turnover#94367891, CASE WHEN ((numcos#94367717 = NA) OR (numcos#94367717 = null)) THEN null ELSE cast(numcos#94367717 as float) END AS numcos#94367892, CASE WHEN ((benchmark#94367718 = NA) OR (benchmark#94367718 = null)) THEN null ELSE cast(benchmark#94367718 as fl... [3624178] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624179 [labelType="html" label="MapPartitionsRDD [3624179] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624180 [labelType="html" label="MapPartitionsRDD [3624180] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624181 [labelType="html" label="MapPartitionsRDD [3624181] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302137 { label="Scan csv "; 3624182 [labelType="html" label="FileScanRDD [3624182] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624183 [labelType="html" label="MapPartitionsRDD [3624183] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302134 { label="WholeStageCodegen (1)"; 3624184 [labelType="html" label="MapPartitionsRDD [3624184] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302133 { label="InMemoryTableScan"; 3624185 [labelType="html" label="MapPartitionsRDD [3624185] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624186 [labelType="html" label="*(1) Project [CASE WHEN (date#94367734 = null) THEN null ELSE cast(date#94367734 as date) END AS date#94368102, CASE WHEN ((cap#94367735 = NA) OR (cap#94367735 = null)) THEN null ELSE cast(cap#94367735 as float) END AS cap#94368103, CASE WHEN ((ret#94367736 = NA) OR (ret#94367736 = null)) THEN null ELSE cast(ret#94367736 as float) END AS ret#94368104, CASE WHEN ((resret#94367737 = NA) OR (resret#94367737 = null)) THEN null ELSE cast(resret#94367737 as float) END AS resret#94368123, CASE WHEN ((retnet#94367738 = NA) OR (retnet#94367738 = null)) THEN null ELSE cast(retnet#94367738 as float) END AS retnet#94368124, CASE WHEN ((turnover#94367739 = NA) OR (turnover#94367739 = null)) THEN null ELSE cast(turnover#94367739 as float) END AS turnover#94368125, CASE WHEN ((numcos#94367740 = NA) OR (numcos#94367740 = null)) THEN null ELSE cast(numcos#94367740 as float) END AS numcos#94368126, CASE WHEN ((coverage#94367741 = NA) OR (coverage#94367741 = null)) THEN null ELSE cast(coverage#94367741 as float) END AS cover... [3624186] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624187 [labelType="html" label="MapPartitionsRDD [3624187] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624188 [labelType="html" label="MapPartitionsRDD [3624188] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624189 [labelType="html" label="MapPartitionsRDD [3624189] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302129 { label="WholeStageCodegen (1)"; 3624190 [labelType="html" label="MapPartitionsRDD [3624190] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302123 { label="Union"; 3624191 [labelType="html" label="UnionRDD [3624191] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302122 { label="Exchange"; 3624195 [labelType="html" label="MapPartitionsRDD [3624195] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624174->3624175; 3624175->3624176; 3624176->3624177; 3624177->3624178; 3624178->3624179; 3624179->3624180; 3624180->3624181; 3624182->3624183; 3624183->3624184; 3624184->3624185; 3624185->3624186; 3624186->3624187; 3624187->3624188; 3624188->3624189; 3624189->3624190; 3624181->3624191; 3624190->3624191; 3624191->3624195; }

3624178

3624186

digraph G { subgraph clusterstage_512971 { label="Stage 512971"; subgraph cluster6302122 { label="Exchange"; 3624196 [labelType="html" label="ShuffledRowRDD [3624196] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302119 { label="WholeStageCodegen (2)"; 3624197 [labelType="html" label="MapPartitionsRDD [3624197] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302118 { label="InMemoryTableScan"; 3624198 [labelType="html" label="MapPartitionsRDD [3624198] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624199 [labelType="html" label="*(2) Sort [date#94367886 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94367886 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7534611]\n +- Union\n :- InMemoryTableScan [date#94367886, numcos#94367892]\n : +- InMemoryRelation [date#94367886, overall#94367887, ret#94367888, resret#94367889, retnet#94367890, turnover#94367891, numcos#94367892, benchmark#94367893, excess_ret#94367894, excess_resret#94367895, excess_retnet#94367896], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94367711 = null) THEN null ELSE cast(date#94367711 as date) END AS date#94367886, CASE WHEN ((overall#94367712 = NA) OR (overall#94367712 = null)) THEN null ELSE cast(overall#94367712 as int) END AS overall#94367887, CASE WHEN ((ret#94367713 = NA) OR (ret#94367713 = null)) THEN null ELSE cast(ret#94367713 as float) END AS ret#94367888, CASE WHEN ((resret#94367714 = NA) OR (resret#94367714 = null)) THEN null ELSE cast(resret#94367714 as float) END A... [3624199] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624200 [labelType="html" label="MapPartitionsRDD [3624200] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624201 [labelType="html" label="MapPartitionsRDD [3624201] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624202 [labelType="html" label="MapPartitionsRDD [3624202] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302117 { label="CollectLimit"; 3624203 [labelType="html" label="MapPartitionsRDD [3624203] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624204 [labelType="html" label="MapPartitionsRDD [3624204] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624196->3624197; 3624197->3624198; 3624198->3624199; 3624199->3624200; 3624200->3624201; 3624201->3624202; 3624202->3624203; 3624203->3624204; }

3624195,3624196

3624199

digraph G { subgraph clusterstage_512972 { label="Stage 512972"; subgraph cluster6302117 { label="CollectLimit"; 3624205 [labelType="html" label="ShuffledRowRDD [3624205] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624206 [labelType="html" label="MapPartitionsRDD [3624206] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302159 { label="mapPartitionsInternal"; 3624207 [labelType="html" label="MapPartitionsRDD [3624207] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302160 { label="map"; 3624208 [labelType="html" label="MapPartitionsRDD [3624208] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624205->3624206; 3624206->3624207; 3624207->3624208; }

3624204,3624205

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512972	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:22:26	20 ms	1/1		56.4 KiB
512971	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94367886 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94367886 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7534611] +- Union :- InMemoryTableScan [date#94367886, numcos#94367892] : +- InMemoryRelation [date#94367886, overall#94367887, ret#94367888, resret#94367889, retnet#94367890, turnover#94367891, numcos#94367892, benchmark#94367893, excess_ret#94367894, excess_resret#94367895, excess_retnet#94367896], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94367711 = null) THEN null ELSE cast(date#94367711 as date) END AS date#94367886, CASE WHEN ((overall#94367712 = NA) OR (overall#94367712 = null)) THEN null ELSE cast(overall#94367712 as int) END AS overall#94367887, CASE WHEN ((ret#94367713 = NA) OR (ret#94367713 = null)) THEN null ELSE cast(ret#94367713 as float) END AS ret#94367888, CASE WHEN ((resret#94367714 = NA) OR (resret#94367714 = null)) THEN null ELSE cast(resret#94367714 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:22:26	0.8 s	200/200		56.4 KiB	56.4 KiB
512970	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94367711 = null) THEN null ELSE cast(date#94367711 as date) END AS date#94367886, CASE WHEN ((overall#94367712 = NA) OR (overall#94367712 = null)) THEN null ELSE cast(overall#94367712 as int) END AS overall#94367887, CASE WHEN ((ret#94367713 = NA) OR (ret#94367713 = null)) THEN null ELSE cast(ret#94367713 as float) END AS ret#94367888, CASE WHEN ((resret#94367714 = NA) OR (resret#94367714 = null)) THEN null ELSE cast(resret#94367714 as float) END AS resret#94367889, CASE WHEN ((retnet#94367715 = NA) OR (retnet#94367715 = null)) THEN null ELSE cast(retnet#94367715 as float) END AS retnet#94367890, CASE WHEN ((turnover#94367716 = NA) OR (turnover#94367716 = null)) THEN null ELSE cast(turnover#94367716 as float) END AS turnover#94367891, CASE WHEN ((numcos#94367717 = NA) OR (numcos#94367717 = null)) THEN null ELSE cast(numcos#94367717 as float) END AS numcos#94367892, CASE WHEN ((benchmark#94367718 = NA) OR (benchmark#94367718 = null)) THEN null ELSE cast(benchmark#94367718 as fl...(1) Project [CASE WHEN (date#94367734 = null) THEN null ELSE cast(date#94367734 as date) END AS date#94368102, CASE WHEN ((cap#94367735 = NA) OR (cap#94367735 = null)) THEN null ELSE cast(cap#94367735 as float) END AS cap#94368103, CASE WHEN ((ret#94367736 = NA) OR (ret#94367736 = null)) THEN null ELSE cast(ret#94367736 as float) END AS ret#94368104, CASE WHEN ((resret#94367737 = NA) OR (resret#94367737 = null)) THEN null ELSE cast(resret#94367737 as float) END AS resret#94368123, CASE WHEN ((retnet#94367738 = NA) OR (retnet#94367738 = null)) THEN null ELSE cast(retnet#94367738 as float) END AS retnet#94368124, CASE WHEN ((turnover#94367739 = NA) OR (turnover#94367739 = null)) THEN null ELSE cast(turnover#94367739 as float) END AS turnover#94368125, CASE WHEN ((numcos#94367740 = NA) OR (numcos#94367740 = null)) THEN null ELSE cast(numcos#94367740 as float) END AS numcos#94368126, CASE WHEN ((coverage#94367741 = NA) OR (coverage#94367741 = null)) THEN null ELSE cast(coverage#94367741 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:22:26	58 ms	2/2	1084.8 KiB		56.4 KiB

Page: