PlusAMP App Middleware - Details for Job 331012

Status: SUCCEEDED
Submitted: 2025/08/02 15:08:40
Duration: 0.8 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512378 { label="Stage 512378"; subgraph cluster6294626 { label="Scan csv "; 3619854 [labelType="html" label="FileScanRDD [3619854] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619855 [labelType="html" label="MapPartitionsRDD [3619855] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294623 { label="WholeStageCodegen (1)"; 3619856 [labelType="html" label="MapPartitionsRDD [3619856] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294622 { label="InMemoryTableScan"; 3619857 [labelType="html" label="MapPartitionsRDD [3619857] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619858 [labelType="html" label="*(1) Project [CASE WHEN (date#94254675 = null) THEN null ELSE cast(date#94254675 as date) END AS date#94254826, CASE WHEN ((overall#94254676 = NA) OR (overall#94254676 = null)) THEN null ELSE cast(overall#94254676 as int) END AS overall#94254827, CASE WHEN ((ret#94254677 = NA) OR (ret#94254677 = null)) THEN null ELSE cast(ret#94254677 as float) END AS ret#94254828, CASE WHEN ((resret#94254678 = NA) OR (resret#94254678 = null)) THEN null ELSE cast(resret#94254678 as float) END AS resret#94254829, CASE WHEN ((retnet#94254679 = NA) OR (retnet#94254679 = null)) THEN null ELSE cast(retnet#94254679 as float) END AS retnet#94254830, CASE WHEN ((turnover#94254680 = NA) OR (turnover#94254680 = null)) THEN null ELSE cast(turnover#94254680 as float) END AS turnover#94254831, CASE WHEN ((numcos#94254681 = NA) OR (numcos#94254681 = null)) THEN null ELSE cast(numcos#94254681 as float) END AS numcos#94254832, CASE WHEN ((benchmark#94254682 = NA) OR (benchmark#94254682 = null)) THEN null ELSE cast(benchmark#94254682 as fl... [3619858] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619859 [labelType="html" label="MapPartitionsRDD [3619859] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619860 [labelType="html" label="MapPartitionsRDD [3619860] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619861 [labelType="html" label="MapPartitionsRDD [3619861] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294635 { label="Scan csv "; 3619862 [labelType="html" label="FileScanRDD [3619862] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619863 [labelType="html" label="MapPartitionsRDD [3619863] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294632 { label="WholeStageCodegen (1)"; 3619864 [labelType="html" label="MapPartitionsRDD [3619864] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294631 { label="InMemoryTableScan"; 3619865 [labelType="html" label="MapPartitionsRDD [3619865] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619866 [labelType="html" label="*(1) Project [CASE WHEN (date#94254649 = null) THEN null ELSE cast(date#94254649 as date) END AS date#94254913, CASE WHEN ((cap#94254650 = NA) OR (cap#94254650 = null)) THEN null ELSE cast(cap#94254650 as float) END AS cap#94254914, CASE WHEN ((ret#94254651 = NA) OR (ret#94254651 = null)) THEN null ELSE cast(ret#94254651 as float) END AS ret#94254942, CASE WHEN ((resret#94254652 = NA) OR (resret#94254652 = null)) THEN null ELSE cast(resret#94254652 as float) END AS resret#94255044, CASE WHEN ((retnet#94254653 = NA) OR (retnet#94254653 = null)) THEN null ELSE cast(retnet#94254653 as float) END AS retnet#94255047, CASE WHEN ((turnover#94254654 = NA) OR (turnover#94254654 = null)) THEN null ELSE cast(turnover#94254654 as float) END AS turnover#94255050, CASE WHEN ((numcos#94254655 = NA) OR (numcos#94254655 = null)) THEN null ELSE cast(numcos#94254655 as float) END AS numcos#94255053, CASE WHEN ((coverage#94254656 = NA) OR (coverage#94254656 = null)) THEN null ELSE cast(coverage#94254656 as float) END AS cover... [3619866] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619867 [labelType="html" label="MapPartitionsRDD [3619867] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619868 [labelType="html" label="MapPartitionsRDD [3619868] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619869 [labelType="html" label="MapPartitionsRDD [3619869] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294627 { label="WholeStageCodegen (1)"; 3619870 [labelType="html" label="MapPartitionsRDD [3619870] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294621 { label="Union"; 3619871 [labelType="html" label="UnionRDD [3619871] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294620 { label="Exchange"; 3619875 [labelType="html" label="MapPartitionsRDD [3619875] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3619854->3619855; 3619855->3619856; 3619856->3619857; 3619857->3619858; 3619858->3619859; 3619859->3619860; 3619860->3619861; 3619862->3619863; 3619863->3619864; 3619864->3619865; 3619865->3619866; 3619866->3619867; 3619867->3619868; 3619868->3619869; 3619869->3619870; 3619861->3619871; 3619870->3619871; 3619871->3619875; }

3619858

3619866

digraph G { subgraph clusterstage_512379 { label="Stage 512379"; subgraph cluster6294620 { label="Exchange"; 3619876 [labelType="html" label="ShuffledRowRDD [3619876] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294617 { label="WholeStageCodegen (2)"; 3619877 [labelType="html" label="MapPartitionsRDD [3619877] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294616 { label="InMemoryTableScan"; 3619878 [labelType="html" label="MapPartitionsRDD [3619878] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619879 [labelType="html" label="*(2) Sort [date#94254826 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94254826 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7525489]\n +- Union\n :- InMemoryTableScan [date#94254826, numcos#94254832]\n : +- InMemoryRelation [date#94254826, overall#94254827, ret#94254828, resret#94254829, retnet#94254830, turnover#94254831, numcos#94254832, benchmark#94254833, excess_ret#94254834, excess_resret#94254835, excess_retnet#94254874], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94254675 = null) THEN null ELSE cast(date#94254675 as date) END AS date#94254826, CASE WHEN ((overall#94254676 = NA) OR (overall#94254676 = null)) THEN null ELSE cast(overall#94254676 as int) END AS overall#94254827, CASE WHEN ((ret#94254677 = NA) OR (ret#94254677 = null)) THEN null ELSE cast(ret#94254677 as float) END AS ret#94254828, CASE WHEN ((resret#94254678 = NA) OR (resret#94254678 = null)) THEN null ELSE cast(resret#94254678 as float) END A... [3619879] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619880 [labelType="html" label="MapPartitionsRDD [3619880] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619881 [labelType="html" label="MapPartitionsRDD [3619881] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619882 [labelType="html" label="MapPartitionsRDD [3619882] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294615 { label="CollectLimit"; 3619883 [labelType="html" label="MapPartitionsRDD [3619883] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619884 [labelType="html" label="MapPartitionsRDD [3619884] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3619876->3619877; 3619877->3619878; 3619878->3619879; 3619879->3619880; 3619880->3619881; 3619881->3619882; 3619882->3619883; 3619883->3619884; }

3619875,3619876

3619879

digraph G { subgraph clusterstage_512380 { label="Stage 512380"; subgraph cluster6294615 { label="CollectLimit"; 3619885 [labelType="html" label="ShuffledRowRDD [3619885] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3619886 [labelType="html" label="MapPartitionsRDD [3619886] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294657 { label="mapPartitionsInternal"; 3619887 [labelType="html" label="MapPartitionsRDD [3619887] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6294658 { label="map"; 3619888 [labelType="html" label="MapPartitionsRDD [3619888] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3619885->3619886; 3619886->3619887; 3619887->3619888; }

3619884,3619885

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512380	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:08:41	19 ms	1/1		51.3 KiB
512379	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94254826 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94254826 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7525489] +- Union :- InMemoryTableScan [date#94254826, numcos#94254832] : +- InMemoryRelation [date#94254826, overall#94254827, ret#94254828, resret#94254829, retnet#94254830, turnover#94254831, numcos#94254832, benchmark#94254833, excess_ret#94254834, excess_resret#94254835, excess_retnet#94254874], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94254675 = null) THEN null ELSE cast(date#94254675 as date) END AS date#94254826, CASE WHEN ((overall#94254676 = NA) OR (overall#94254676 = null)) THEN null ELSE cast(overall#94254676 as int) END AS overall#94254827, CASE WHEN ((ret#94254677 = NA) OR (ret#94254677 = null)) THEN null ELSE cast(ret#94254677 as float) END AS ret#94254828, CASE WHEN ((resret#94254678 = NA) OR (resret#94254678 = null)) THEN null ELSE cast(resret#94254678 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:08:40	0.7 s	200/200		51.3 KiB	51.3 KiB
512378	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94254675 = null) THEN null ELSE cast(date#94254675 as date) END AS date#94254826, CASE WHEN ((overall#94254676 = NA) OR (overall#94254676 = null)) THEN null ELSE cast(overall#94254676 as int) END AS overall#94254827, CASE WHEN ((ret#94254677 = NA) OR (ret#94254677 = null)) THEN null ELSE cast(ret#94254677 as float) END AS ret#94254828, CASE WHEN ((resret#94254678 = NA) OR (resret#94254678 = null)) THEN null ELSE cast(resret#94254678 as float) END AS resret#94254829, CASE WHEN ((retnet#94254679 = NA) OR (retnet#94254679 = null)) THEN null ELSE cast(retnet#94254679 as float) END AS retnet#94254830, CASE WHEN ((turnover#94254680 = NA) OR (turnover#94254680 = null)) THEN null ELSE cast(turnover#94254680 as float) END AS turnover#94254831, CASE WHEN ((numcos#94254681 = NA) OR (numcos#94254681 = null)) THEN null ELSE cast(numcos#94254681 as float) END AS numcos#94254832, CASE WHEN ((benchmark#94254682 = NA) OR (benchmark#94254682 = null)) THEN null ELSE cast(benchmark#94254682 as fl...(1) Project [CASE WHEN (date#94254649 = null) THEN null ELSE cast(date#94254649 as date) END AS date#94254913, CASE WHEN ((cap#94254650 = NA) OR (cap#94254650 = null)) THEN null ELSE cast(cap#94254650 as float) END AS cap#94254914, CASE WHEN ((ret#94254651 = NA) OR (ret#94254651 = null)) THEN null ELSE cast(ret#94254651 as float) END AS ret#94254942, CASE WHEN ((resret#94254652 = NA) OR (resret#94254652 = null)) THEN null ELSE cast(resret#94254652 as float) END AS resret#94255044, CASE WHEN ((retnet#94254653 = NA) OR (retnet#94254653 = null)) THEN null ELSE cast(retnet#94254653 as float) END AS retnet#94255047, CASE WHEN ((turnover#94254654 = NA) OR (turnover#94254654 = null)) THEN null ELSE cast(turnover#94254654 as float) END AS turnover#94255050, CASE WHEN ((numcos#94254655 = NA) OR (numcos#94254655 = null)) THEN null ELSE cast(numcos#94254655 as float) END AS numcos#94255053, CASE WHEN ((coverage#94254656 = NA) OR (coverage#94254656 = null)) THEN null ELSE cast(coverage#94254656 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:08:40	41 ms	2/2	850.0 KiB		51.3 KiB

Page: