PlusAMP App Middleware - Details for Job 331453

Status: SUCCEEDED
Submitted: 2025/08/02 15:24:09
Duration: 1 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_513044 { label="Stage 513044"; subgraph cluster6303061 { label="Scan csv "; 3624711 [labelType="html" label="FileScanRDD [3624711] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624712 [labelType="html" label="MapPartitionsRDD [3624712] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303058 { label="WholeStageCodegen (1)"; 3624713 [labelType="html" label="MapPartitionsRDD [3624713] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303057 { label="InMemoryTableScan"; 3624714 [labelType="html" label="MapPartitionsRDD [3624714] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624715 [labelType="html" label="*(1) Project [CASE WHEN (date#94382301 = null) THEN null ELSE cast(date#94382301 as date) END AS date#94382484, CASE WHEN ((overall#94382302 = NA) OR (overall#94382302 = null)) THEN null ELSE cast(overall#94382302 as int) END AS overall#94382488, CASE WHEN ((ret#94382303 = NA) OR (ret#94382303 = null)) THEN null ELSE cast(ret#94382303 as float) END AS ret#94382490, CASE WHEN ((resret#94382304 = NA) OR (resret#94382304 = null)) THEN null ELSE cast(resret#94382304 as float) END AS resret#94382518, CASE WHEN ((retnet#94382305 = NA) OR (retnet#94382305 = null)) THEN null ELSE cast(retnet#94382305 as float) END AS retnet#94382519, CASE WHEN ((turnover#94382306 = NA) OR (turnover#94382306 = null)) THEN null ELSE cast(turnover#94382306 as float) END AS turnover#94382546, CASE WHEN ((numcos#94382307 = NA) OR (numcos#94382307 = null)) THEN null ELSE cast(numcos#94382307 as float) END AS numcos#94382547, CASE WHEN ((benchmark#94382308 = NA) OR (benchmark#94382308 = null)) THEN null ELSE cast(benchmark#94382308 as fl... [3624715] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624716 [labelType="html" label="MapPartitionsRDD [3624716] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624717 [labelType="html" label="MapPartitionsRDD [3624717] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624718 [labelType="html" label="MapPartitionsRDD [3624718] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303070 { label="Scan csv "; 3624719 [labelType="html" label="FileScanRDD [3624719] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624720 [labelType="html" label="MapPartitionsRDD [3624720] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303067 { label="WholeStageCodegen (1)"; 3624721 [labelType="html" label="MapPartitionsRDD [3624721] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303066 { label="InMemoryTableScan"; 3624722 [labelType="html" label="MapPartitionsRDD [3624722] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624723 [labelType="html" label="*(1) Project [CASE WHEN (date#94382274 = null) THEN null ELSE cast(date#94382274 as date) END AS date#94382455, CASE WHEN ((cap#94382275 = NA) OR (cap#94382275 = null)) THEN null ELSE cast(cap#94382275 as float) END AS cap#94382456, CASE WHEN ((ret#94382276 = NA) OR (ret#94382276 = null)) THEN null ELSE cast(ret#94382276 as float) END AS ret#94382468, CASE WHEN ((resret#94382277 = NA) OR (resret#94382277 = null)) THEN null ELSE cast(resret#94382277 as float) END AS resret#94382469, CASE WHEN ((retnet#94382278 = NA) OR (retnet#94382278 = null)) THEN null ELSE cast(retnet#94382278 as float) END AS retnet#94382481, CASE WHEN ((turnover#94382279 = NA) OR (turnover#94382279 = null)) THEN null ELSE cast(turnover#94382279 as float) END AS turnover#94382482, CASE WHEN ((numcos#94382280 = NA) OR (numcos#94382280 = null)) THEN null ELSE cast(numcos#94382280 as float) END AS numcos#94382483, CASE WHEN ((coverage#94382281 = NA) OR (coverage#94382281 = null)) THEN null ELSE cast(coverage#94382281 as float) END AS cover... [3624723] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624724 [labelType="html" label="MapPartitionsRDD [3624724] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624725 [labelType="html" label="MapPartitionsRDD [3624725] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624726 [labelType="html" label="MapPartitionsRDD [3624726] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303062 { label="WholeStageCodegen (1)"; 3624727 [labelType="html" label="MapPartitionsRDD [3624727] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303056 { label="Union"; 3624728 [labelType="html" label="UnionRDD [3624728] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303055 { label="Exchange"; 3624732 [labelType="html" label="MapPartitionsRDD [3624732] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624711->3624712; 3624712->3624713; 3624713->3624714; 3624714->3624715; 3624715->3624716; 3624716->3624717; 3624717->3624718; 3624719->3624720; 3624720->3624721; 3624721->3624722; 3624722->3624723; 3624723->3624724; 3624724->3624725; 3624725->3624726; 3624726->3624727; 3624718->3624728; 3624727->3624728; 3624728->3624732; }

3624715

3624723

digraph G { subgraph clusterstage_513045 { label="Stage 513045"; subgraph cluster6303055 { label="Exchange"; 3624733 [labelType="html" label="ShuffledRowRDD [3624733] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303052 { label="WholeStageCodegen (2)"; 3624734 [labelType="html" label="MapPartitionsRDD [3624734] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303051 { label="InMemoryTableScan"; 3624735 [labelType="html" label="MapPartitionsRDD [3624735] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624736 [labelType="html" label="*(2) Sort [date#94382484 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94382484 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535759]\n +- Union\n :- InMemoryTableScan [date#94382484, numcos#94382547]\n : +- InMemoryRelation [date#94382484, overall#94382488, ret#94382490, resret#94382518, retnet#94382519, turnover#94382546, numcos#94382547, benchmark#94382548, excess_ret#94382549, excess_resret#94382564, excess_retnet#94382566], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94382301 = null) THEN null ELSE cast(date#94382301 as date) END AS date#94382484, CASE WHEN ((overall#94382302 = NA) OR (overall#94382302 = null)) THEN null ELSE cast(overall#94382302 as int) END AS overall#94382488, CASE WHEN ((ret#94382303 = NA) OR (ret#94382303 = null)) THEN null ELSE cast(ret#94382303 as float) END AS ret#94382490, CASE WHEN ((resret#94382304 = NA) OR (resret#94382304 = null)) THEN null ELSE cast(resret#94382304 as float) END A... [3624736] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624737 [labelType="html" label="MapPartitionsRDD [3624737] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624738 [labelType="html" label="MapPartitionsRDD [3624738] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624739 [labelType="html" label="MapPartitionsRDD [3624739] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303050 { label="CollectLimit"; 3624740 [labelType="html" label="MapPartitionsRDD [3624740] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624741 [labelType="html" label="MapPartitionsRDD [3624741] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624733->3624734; 3624734->3624735; 3624735->3624736; 3624736->3624737; 3624737->3624738; 3624738->3624739; 3624739->3624740; 3624740->3624741; }

3624732,3624733

3624736

digraph G { subgraph clusterstage_513046 { label="Stage 513046"; subgraph cluster6303050 { label="CollectLimit"; 3624742 [labelType="html" label="ShuffledRowRDD [3624742] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624743 [labelType="html" label="MapPartitionsRDD [3624743] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303092 { label="mapPartitionsInternal"; 3624744 [labelType="html" label="MapPartitionsRDD [3624744] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303093 { label="map"; 3624745 [labelType="html" label="MapPartitionsRDD [3624745] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624742->3624743; 3624743->3624744; 3624744->3624745; }

3624741,3624742

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
513046	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:24:11	34 ms	1/1		59.7 KiB
513045	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94382484 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94382484 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535759] +- Union :- InMemoryTableScan [date#94382484, numcos#94382547] : +- InMemoryRelation [date#94382484, overall#94382488, ret#94382490, resret#94382518, retnet#94382519, turnover#94382546, numcos#94382547, benchmark#94382548, excess_ret#94382549, excess_resret#94382564, excess_retnet#94382566], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94382301 = null) THEN null ELSE cast(date#94382301 as date) END AS date#94382484, CASE WHEN ((overall#94382302 = NA) OR (overall#94382302 = null)) THEN null ELSE cast(overall#94382302 as int) END AS overall#94382488, CASE WHEN ((ret#94382303 = NA) OR (ret#94382303 = null)) THEN null ELSE cast(ret#94382303 as float) END AS ret#94382490, CASE WHEN ((resret#94382304 = NA) OR (resret#94382304 = null)) THEN null ELSE cast(resret#94382304 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:24:10	1 s	200/200		59.7 KiB	59.7 KiB
513044	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94382301 = null) THEN null ELSE cast(date#94382301 as date) END AS date#94382484, CASE WHEN ((overall#94382302 = NA) OR (overall#94382302 = null)) THEN null ELSE cast(overall#94382302 as int) END AS overall#94382488, CASE WHEN ((ret#94382303 = NA) OR (ret#94382303 = null)) THEN null ELSE cast(ret#94382303 as float) END AS ret#94382490, CASE WHEN ((resret#94382304 = NA) OR (resret#94382304 = null)) THEN null ELSE cast(resret#94382304 as float) END AS resret#94382518, CASE WHEN ((retnet#94382305 = NA) OR (retnet#94382305 = null)) THEN null ELSE cast(retnet#94382305 as float) END AS retnet#94382519, CASE WHEN ((turnover#94382306 = NA) OR (turnover#94382306 = null)) THEN null ELSE cast(turnover#94382306 as float) END AS turnover#94382546, CASE WHEN ((numcos#94382307 = NA) OR (numcos#94382307 = null)) THEN null ELSE cast(numcos#94382307 as float) END AS numcos#94382547, CASE WHEN ((benchmark#94382308 = NA) OR (benchmark#94382308 = null)) THEN null ELSE cast(benchmark#94382308 as fl...(1) Project [CASE WHEN (date#94382274 = null) THEN null ELSE cast(date#94382274 as date) END AS date#94382455, CASE WHEN ((cap#94382275 = NA) OR (cap#94382275 = null)) THEN null ELSE cast(cap#94382275 as float) END AS cap#94382456, CASE WHEN ((ret#94382276 = NA) OR (ret#94382276 = null)) THEN null ELSE cast(ret#94382276 as float) END AS ret#94382468, CASE WHEN ((resret#94382277 = NA) OR (resret#94382277 = null)) THEN null ELSE cast(resret#94382277 as float) END AS resret#94382469, CASE WHEN ((retnet#94382278 = NA) OR (retnet#94382278 = null)) THEN null ELSE cast(retnet#94382278 as float) END AS retnet#94382481, CASE WHEN ((turnover#94382279 = NA) OR (turnover#94382279 = null)) THEN null ELSE cast(turnover#94382279 as float) END AS turnover#94382482, CASE WHEN ((numcos#94382280 = NA) OR (numcos#94382280 = null)) THEN null ELSE cast(numcos#94382280 as float) END AS numcos#94382483, CASE WHEN ((coverage#94382281 = NA) OR (coverage#94382281 = null)) THEN null ELSE cast(coverage#94382281 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:24:09	58 ms	2/2	1084.8 KiB		59.7 KiB

Page: