PlusAMP App Middleware - Details for Job 330550

Status: SUCCEEDED
Submitted: 2025/08/01 16:00:11
Duration: 0.9 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_511650 { label="Stage 511650"; subgraph cluster6285898 { label="Scan csv "; 3614866 [labelType="html" label="FileScanRDD [3614866] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614867 [labelType="html" label="MapPartitionsRDD [3614867] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285895 { label="WholeStageCodegen (1)"; 3614868 [labelType="html" label="MapPartitionsRDD [3614868] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285894 { label="InMemoryTableScan"; 3614869 [labelType="html" label="MapPartitionsRDD [3614869] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614870 [labelType="html" label="*(1) Project [CASE WHEN (date#94125102 = null) THEN null ELSE cast(date#94125102 as date) END AS date#94125225, CASE WHEN ((overall#94125103 = NA) OR (overall#94125103 = null)) THEN null ELSE cast(overall#94125103 as int) END AS overall#94125226, CASE WHEN ((ret#94125104 = NA) OR (ret#94125104 = null)) THEN null ELSE cast(ret#94125104 as float) END AS ret#94125253, CASE WHEN ((resret#94125105 = NA) OR (resret#94125105 = null)) THEN null ELSE cast(resret#94125105 as float) END AS resret#94125254, CASE WHEN ((retnet#94125106 = NA) OR (retnet#94125106 = null)) THEN null ELSE cast(retnet#94125106 as float) END AS retnet#94125281, CASE WHEN ((turnover#94125107 = NA) OR (turnover#94125107 = null)) THEN null ELSE cast(turnover#94125107 as float) END AS turnover#94125294, CASE WHEN ((numcos#94125108 = NA) OR (numcos#94125108 = null)) THEN null ELSE cast(numcos#94125108 as float) END AS numcos#94125295, CASE WHEN ((benchmark#94125109 = NA) OR (benchmark#94125109 = null)) THEN null ELSE cast(benchmark#94125109 as fl... [3614870] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614871 [labelType="html" label="MapPartitionsRDD [3614871] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614872 [labelType="html" label="MapPartitionsRDD [3614872] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614873 [labelType="html" label="MapPartitionsRDD [3614873] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285907 { label="Scan csv "; 3614874 [labelType="html" label="FileScanRDD [3614874] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614875 [labelType="html" label="MapPartitionsRDD [3614875] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285904 { label="WholeStageCodegen (1)"; 3614876 [labelType="html" label="MapPartitionsRDD [3614876] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285903 { label="InMemoryTableScan"; 3614877 [labelType="html" label="MapPartitionsRDD [3614877] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614878 [labelType="html" label="*(1) Project [CASE WHEN (date#94125124 = null) THEN null ELSE cast(date#94125124 as date) END AS date#94125311, CASE WHEN ((cap#94125125 = NA) OR (cap#94125125 = null)) THEN null ELSE cast(cap#94125125 as float) END AS cap#94125313, CASE WHEN ((ret#94125126 = NA) OR (ret#94125126 = null)) THEN null ELSE cast(ret#94125126 as float) END AS ret#94125314, CASE WHEN ((resret#94125127 = NA) OR (resret#94125127 = null)) THEN null ELSE cast(resret#94125127 as float) END AS resret#94125341, CASE WHEN ((retnet#94125128 = NA) OR (retnet#94125128 = null)) THEN null ELSE cast(retnet#94125128 as float) END AS retnet#94125342, CASE WHEN ((turnover#94125129 = NA) OR (turnover#94125129 = null)) THEN null ELSE cast(turnover#94125129 as float) END AS turnover#94125343, CASE WHEN ((numcos#94125130 = NA) OR (numcos#94125130 = null)) THEN null ELSE cast(numcos#94125130 as float) END AS numcos#94125370, CASE WHEN ((coverage#94125131 = NA) OR (coverage#94125131 = null)) THEN null ELSE cast(coverage#94125131 as float) END AS cover... [3614878] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614879 [labelType="html" label="MapPartitionsRDD [3614879] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614880 [labelType="html" label="MapPartitionsRDD [3614880] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614881 [labelType="html" label="MapPartitionsRDD [3614881] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285899 { label="WholeStageCodegen (1)"; 3614882 [labelType="html" label="MapPartitionsRDD [3614882] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285893 { label="Union"; 3614883 [labelType="html" label="UnionRDD [3614883] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285892 { label="Exchange"; 3614887 [labelType="html" label="MapPartitionsRDD [3614887] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3614866->3614867; 3614867->3614868; 3614868->3614869; 3614869->3614870; 3614870->3614871; 3614871->3614872; 3614872->3614873; 3614874->3614875; 3614875->3614876; 3614876->3614877; 3614877->3614878; 3614878->3614879; 3614879->3614880; 3614880->3614881; 3614881->3614882; 3614873->3614883; 3614882->3614883; 3614883->3614887; }

3614870

3614878

digraph G { subgraph clusterstage_511651 { label="Stage 511651"; subgraph cluster6285892 { label="Exchange"; 3614888 [labelType="html" label="ShuffledRowRDD [3614888] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285889 { label="WholeStageCodegen (2)"; 3614889 [labelType="html" label="MapPartitionsRDD [3614889] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285888 { label="InMemoryTableScan"; 3614890 [labelType="html" label="MapPartitionsRDD [3614890] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614891 [labelType="html" label="*(2) Sort [date#94125225 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94125225 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7515277]\n +- Union\n :- InMemoryTableScan [date#94125225, numcos#94125295]\n : +- InMemoryRelation [date#94125225, overall#94125226, ret#94125253, resret#94125254, retnet#94125281, turnover#94125294, numcos#94125295, benchmark#94125308, excess_ret#94125309, excess_resret#94125310, excess_retnet#94125312], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94125102 = null) THEN null ELSE cast(date#94125102 as date) END AS date#94125225, CASE WHEN ((overall#94125103 = NA) OR (overall#94125103 = null)) THEN null ELSE cast(overall#94125103 as int) END AS overall#94125226, CASE WHEN ((ret#94125104 = NA) OR (ret#94125104 = null)) THEN null ELSE cast(ret#94125104 as float) END AS ret#94125253, CASE WHEN ((resret#94125105 = NA) OR (resret#94125105 = null)) THEN null ELSE cast(resret#94125105 as float) END A... [3614891] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614892 [labelType="html" label="MapPartitionsRDD [3614892] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614893 [labelType="html" label="MapPartitionsRDD [3614893] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614894 [labelType="html" label="MapPartitionsRDD [3614894] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285887 { label="CollectLimit"; 3614895 [labelType="html" label="MapPartitionsRDD [3614895] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614896 [labelType="html" label="MapPartitionsRDD [3614896] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3614888->3614889; 3614889->3614890; 3614890->3614891; 3614891->3614892; 3614892->3614893; 3614893->3614894; 3614894->3614895; 3614895->3614896; }

3614887,3614888

3614891

digraph G { subgraph clusterstage_511652 { label="Stage 511652"; subgraph cluster6285887 { label="CollectLimit"; 3614897 [labelType="html" label="ShuffledRowRDD [3614897] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614898 [labelType="html" label="MapPartitionsRDD [3614898] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285929 { label="mapPartitionsInternal"; 3614899 [labelType="html" label="MapPartitionsRDD [3614899] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285930 { label="map"; 3614900 [labelType="html" label="MapPartitionsRDD [3614900] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3614897->3614898; 3614898->3614899; 3614899->3614900; }

3614896,3614897

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
511652	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 16:00:12	19 ms	1/1		45.9 KiB
511651	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94125225 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94125225 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7515277] +- Union :- InMemoryTableScan [date#94125225, numcos#94125295] : +- InMemoryRelation [date#94125225, overall#94125226, ret#94125253, resret#94125254, retnet#94125281, turnover#94125294, numcos#94125295, benchmark#94125308, excess_ret#94125309, excess_resret#94125310, excess_retnet#94125312], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94125102 = null) THEN null ELSE cast(date#94125102 as date) END AS date#94125225, CASE WHEN ((overall#94125103 = NA) OR (overall#94125103 = null)) THEN null ELSE cast(overall#94125103 as int) END AS overall#94125226, CASE WHEN ((ret#94125104 = NA) OR (ret#94125104 = null)) THEN null ELSE cast(ret#94125104 as float) END AS ret#94125253, CASE WHEN ((resret#94125105 = NA) OR (resret#94125105 = null)) THEN null ELSE cast(resret#94125105 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 16:00:11	0.8 s	200/200		45.9 KiB	45.9 KiB
511650	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94125102 = null) THEN null ELSE cast(date#94125102 as date) END AS date#94125225, CASE WHEN ((overall#94125103 = NA) OR (overall#94125103 = null)) THEN null ELSE cast(overall#94125103 as int) END AS overall#94125226, CASE WHEN ((ret#94125104 = NA) OR (ret#94125104 = null)) THEN null ELSE cast(ret#94125104 as float) END AS ret#94125253, CASE WHEN ((resret#94125105 = NA) OR (resret#94125105 = null)) THEN null ELSE cast(resret#94125105 as float) END AS resret#94125254, CASE WHEN ((retnet#94125106 = NA) OR (retnet#94125106 = null)) THEN null ELSE cast(retnet#94125106 as float) END AS retnet#94125281, CASE WHEN ((turnover#94125107 = NA) OR (turnover#94125107 = null)) THEN null ELSE cast(turnover#94125107 as float) END AS turnover#94125294, CASE WHEN ((numcos#94125108 = NA) OR (numcos#94125108 = null)) THEN null ELSE cast(numcos#94125108 as float) END AS numcos#94125295, CASE WHEN ((benchmark#94125109 = NA) OR (benchmark#94125109 = null)) THEN null ELSE cast(benchmark#94125109 as fl...(1) Project [CASE WHEN (date#94125124 = null) THEN null ELSE cast(date#94125124 as date) END AS date#94125311, CASE WHEN ((cap#94125125 = NA) OR (cap#94125125 = null)) THEN null ELSE cast(cap#94125125 as float) END AS cap#94125313, CASE WHEN ((ret#94125126 = NA) OR (ret#94125126 = null)) THEN null ELSE cast(ret#94125126 as float) END AS ret#94125314, CASE WHEN ((resret#94125127 = NA) OR (resret#94125127 = null)) THEN null ELSE cast(resret#94125127 as float) END AS resret#94125341, CASE WHEN ((retnet#94125128 = NA) OR (retnet#94125128 = null)) THEN null ELSE cast(retnet#94125128 as float) END AS retnet#94125342, CASE WHEN ((turnover#94125129 = NA) OR (turnover#94125129 = null)) THEN null ELSE cast(turnover#94125129 as float) END AS turnover#94125343, CASE WHEN ((numcos#94125130 = NA) OR (numcos#94125130 = null)) THEN null ELSE cast(numcos#94125130 as float) END AS numcos#94125370, CASE WHEN ((coverage#94125131 = NA) OR (coverage#94125131 = null)) THEN null ELSE cast(coverage#94125131 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 16:00:11	45 ms	2/2	819.1 KiB		45.9 KiB

Page: