PlusAMP App Middleware - Details for Job 331355

Status: SUCCEEDED
Submitted: 2025/08/02 15:20:42
Duration: 1.0 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512896 { label="Stage 512896"; subgraph cluster6301189 { label="Scan csv "; 3623631 [labelType="html" label="FileScanRDD [3623631] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623632 [labelType="html" label="MapPartitionsRDD [3623632] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301186 { label="WholeStageCodegen (1)"; 3623633 [labelType="html" label="MapPartitionsRDD [3623633] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301185 { label="InMemoryTableScan"; 3623634 [labelType="html" label="MapPartitionsRDD [3623634] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623635 [labelType="html" label="*(1) Project [CASE WHEN (date#94353924 = null) THEN null ELSE cast(date#94353924 as date) END AS date#94354232, CASE WHEN ((overall#94353925 = NA) OR (overall#94353925 = null)) THEN null ELSE cast(overall#94353925 as int) END AS overall#94354233, CASE WHEN ((ret#94353926 = NA) OR (ret#94353926 = null)) THEN null ELSE cast(ret#94353926 as float) END AS ret#94354234, CASE WHEN ((resret#94353927 = NA) OR (resret#94353927 = null)) THEN null ELSE cast(resret#94353927 as float) END AS resret#94354235, CASE WHEN ((retnet#94353928 = NA) OR (retnet#94353928 = null)) THEN null ELSE cast(retnet#94353928 as float) END AS retnet#94354246, CASE WHEN ((turnover#94353929 = NA) OR (turnover#94353929 = null)) THEN null ELSE cast(turnover#94353929 as float) END AS turnover#94354247, CASE WHEN ((numcos#94353930 = NA) OR (numcos#94353930 = null)) THEN null ELSE cast(numcos#94353930 as float) END AS numcos#94354248, CASE WHEN (benchmark#94353931 = null) THEN null ELSE benchmark#94353931 END AS benchmark#94354249, CASE WHEN (exc... [3623635] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623636 [labelType="html" label="MapPartitionsRDD [3623636] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623637 [labelType="html" label="MapPartitionsRDD [3623637] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623638 [labelType="html" label="MapPartitionsRDD [3623638] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301198 { label="Scan csv "; 3623639 [labelType="html" label="FileScanRDD [3623639] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623640 [labelType="html" label="MapPartitionsRDD [3623640] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301195 { label="WholeStageCodegen (1)"; 3623641 [labelType="html" label="MapPartitionsRDD [3623641] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301194 { label="InMemoryTableScan"; 3623642 [labelType="html" label="MapPartitionsRDD [3623642] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623643 [labelType="html" label="*(1) Project [CASE WHEN (date#94353948 = null) THEN null ELSE cast(date#94353948 as date) END AS date#94354207, CASE WHEN ((cap#94353949 = NA) OR (cap#94353949 = null)) THEN null ELSE cast(cap#94353949 as float) END AS cap#94354209, CASE WHEN ((ret#94353950 = NA) OR (ret#94353950 = null)) THEN null ELSE cast(ret#94353950 as float) END AS ret#94354245, CASE WHEN ((resret#94353951 = NA) OR (resret#94353951 = null)) THEN null ELSE cast(resret#94353951 as float) END AS resret#94354254, CASE WHEN ((retnet#94353952 = NA) OR (retnet#94353952 = null)) THEN null ELSE cast(retnet#94353952 as float) END AS retnet#94354255, CASE WHEN ((turnover#94353953 = NA) OR (turnover#94353953 = null)) THEN null ELSE cast(turnover#94353953 as float) END AS turnover#94354256, CASE WHEN ((numcos#94353954 = NA) OR (numcos#94353954 = null)) THEN null ELSE cast(numcos#94353954 as float) END AS numcos#94354257, CASE WHEN ((coverage#94353955 = NA) OR (coverage#94353955 = null)) THEN null ELSE cast(coverage#94353955 as float) END AS cover... [3623643] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623644 [labelType="html" label="MapPartitionsRDD [3623644] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623645 [labelType="html" label="MapPartitionsRDD [3623645] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623646 [labelType="html" label="MapPartitionsRDD [3623646] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301190 { label="WholeStageCodegen (1)"; 3623647 [labelType="html" label="MapPartitionsRDD [3623647] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301184 { label="Union"; 3623648 [labelType="html" label="UnionRDD [3623648] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301183 { label="Exchange"; 3623652 [labelType="html" label="MapPartitionsRDD [3623652] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623631->3623632; 3623632->3623633; 3623633->3623634; 3623634->3623635; 3623635->3623636; 3623636->3623637; 3623637->3623638; 3623639->3623640; 3623640->3623641; 3623641->3623642; 3623642->3623643; 3623643->3623644; 3623644->3623645; 3623645->3623646; 3623646->3623647; 3623638->3623648; 3623647->3623648; 3623648->3623652; }

3623635

3623643

digraph G { subgraph clusterstage_512897 { label="Stage 512897"; subgraph cluster6301183 { label="Exchange"; 3623653 [labelType="html" label="ShuffledRowRDD [3623653] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301180 { label="WholeStageCodegen (2)"; 3623654 [labelType="html" label="MapPartitionsRDD [3623654] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301179 { label="InMemoryTableScan"; 3623655 [labelType="html" label="MapPartitionsRDD [3623655] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623656 [labelType="html" label="*(2) Sort [date#94354232 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94354232 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7533495]\n +- Union\n :- InMemoryTableScan [date#94354232, numcos#94354248]\n : +- InMemoryRelation [date#94354232, overall#94354233, ret#94354234, resret#94354235, retnet#94354246, turnover#94354247, numcos#94354248, benchmark#94354249, excess_ret#94354250, excess_resret#94354251, excess_retnet#94354252], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94353924 = null) THEN null ELSE cast(date#94353924 as date) END AS date#94354232, CASE WHEN ((overall#94353925 = NA) OR (overall#94353925 = null)) THEN null ELSE cast(overall#94353925 as int) END AS overall#94354233, CASE WHEN ((ret#94353926 = NA) OR (ret#94353926 = null)) THEN null ELSE cast(ret#94353926 as float) END AS ret#94354234, CASE WHEN ((resret#94353927 = NA) OR (resret#94353927 = null)) THEN null ELSE cast(resret#94353927 as float) END A... [3623656] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623657 [labelType="html" label="MapPartitionsRDD [3623657] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623658 [labelType="html" label="MapPartitionsRDD [3623658] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623659 [labelType="html" label="MapPartitionsRDD [3623659] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301178 { label="CollectLimit"; 3623660 [labelType="html" label="MapPartitionsRDD [3623660] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623661 [labelType="html" label="MapPartitionsRDD [3623661] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623653->3623654; 3623654->3623655; 3623655->3623656; 3623656->3623657; 3623657->3623658; 3623658->3623659; 3623659->3623660; 3623660->3623661; }

3623652,3623653

3623656

digraph G { subgraph clusterstage_512898 { label="Stage 512898"; subgraph cluster6301178 { label="CollectLimit"; 3623662 [labelType="html" label="ShuffledRowRDD [3623662] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623663 [labelType="html" label="MapPartitionsRDD [3623663] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301220 { label="mapPartitionsInternal"; 3623664 [labelType="html" label="MapPartitionsRDD [3623664] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6301221 { label="map"; 3623665 [labelType="html" label="MapPartitionsRDD [3623665] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623662->3623663; 3623663->3623664; 3623664->3623665; }

3623661,3623662

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512898	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:43	18 ms	1/1		25.9 KiB
512897	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94354232 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94354232 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7533495] +- Union :- InMemoryTableScan [date#94354232, numcos#94354248] : +- InMemoryRelation [date#94354232, overall#94354233, ret#94354234, resret#94354235, retnet#94354246, turnover#94354247, numcos#94354248, benchmark#94354249, excess_ret#94354250, excess_resret#94354251, excess_retnet#94354252], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94353924 = null) THEN null ELSE cast(date#94353924 as date) END AS date#94354232, CASE WHEN ((overall#94353925 = NA) OR (overall#94353925 = null)) THEN null ELSE cast(overall#94353925 as int) END AS overall#94354233, CASE WHEN ((ret#94353926 = NA) OR (ret#94353926 = null)) THEN null ELSE cast(ret#94353926 as float) END AS ret#94354234, CASE WHEN ((resret#94353927 = NA) OR (resret#94353927 = null)) THEN null ELSE cast(resret#94353927 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:42	0.9 s	200/200		25.9 KiB	25.9 KiB
512896	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94353924 = null) THEN null ELSE cast(date#94353924 as date) END AS date#94354232, CASE WHEN ((overall#94353925 = NA) OR (overall#94353925 = null)) THEN null ELSE cast(overall#94353925 as int) END AS overall#94354233, CASE WHEN ((ret#94353926 = NA) OR (ret#94353926 = null)) THEN null ELSE cast(ret#94353926 as float) END AS ret#94354234, CASE WHEN ((resret#94353927 = NA) OR (resret#94353927 = null)) THEN null ELSE cast(resret#94353927 as float) END AS resret#94354235, CASE WHEN ((retnet#94353928 = NA) OR (retnet#94353928 = null)) THEN null ELSE cast(retnet#94353928 as float) END AS retnet#94354246, CASE WHEN ((turnover#94353929 = NA) OR (turnover#94353929 = null)) THEN null ELSE cast(turnover#94353929 as float) END AS turnover#94354247, CASE WHEN ((numcos#94353930 = NA) OR (numcos#94353930 = null)) THEN null ELSE cast(numcos#94353930 as float) END AS numcos#94354248, CASE WHEN (benchmark#94353931 = null) THEN null ELSE benchmark#94353931 END AS benchmark#94354249, CASE WHEN (exc...(1) Project [CASE WHEN (date#94353948 = null) THEN null ELSE cast(date#94353948 as date) END AS date#94354207, CASE WHEN ((cap#94353949 = NA) OR (cap#94353949 = null)) THEN null ELSE cast(cap#94353949 as float) END AS cap#94354209, CASE WHEN ((ret#94353950 = NA) OR (ret#94353950 = null)) THEN null ELSE cast(ret#94353950 as float) END AS ret#94354245, CASE WHEN ((resret#94353951 = NA) OR (resret#94353951 = null)) THEN null ELSE cast(resret#94353951 as float) END AS resret#94354254, CASE WHEN ((retnet#94353952 = NA) OR (retnet#94353952 = null)) THEN null ELSE cast(retnet#94353952 as float) END AS retnet#94354255, CASE WHEN ((turnover#94353953 = NA) OR (turnover#94353953 = null)) THEN null ELSE cast(turnover#94353953 as float) END AS turnover#94354256, CASE WHEN ((numcos#94353954 = NA) OR (numcos#94353954 = null)) THEN null ELSE cast(numcos#94353954 as float) END AS numcos#94354257, CASE WHEN ((coverage#94353955 = NA) OR (coverage#94353955 = null)) THEN null ELSE cast(coverage#94353955 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:42	48 ms	2/2	335.6 KiB		25.9 KiB

Page: