PlusAMP App Middleware - Details for Job 330536

Status: SUCCEEDED
Submitted: 2025/08/01 15:59:44
Duration: 69 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_511631 { label="Stage 511631"; subgraph cluster6285627 { label="Scan csv "; 3614711 [labelType="html" label="FileScanRDD [3614711] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614712 [labelType="html" label="MapPartitionsRDD [3614712] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285624 { label="WholeStageCodegen (1)"; 3614713 [labelType="html" label="MapPartitionsRDD [3614713] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285623 { label="InMemoryTableScan"; 3614714 [labelType="html" label="MapPartitionsRDD [3614714] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614715 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94119373 = NA) OR (fractile#94119373 = null)) THEN null ELSE cast(fractile#94119373 as int) END AS fractile#94119415, CASE WHEN ((overall#94119374 = NA) OR (overall#94119374 = null)) THEN null ELSE cast(overall#94119374 as int) END AS overall#94119417, CASE WHEN ((annual_ret#94119375 = NA) OR (annual_ret#94119375 = null)) THEN null ELSE cast(annual_ret#94119375 as float) END AS annual_ret#94119441, CASE WHEN ((annual_resret#94119376 = NA) OR (annual_resret#94119376 = null)) THEN null ELSE cast(annual_resret#94119376 as float) END AS annual_resret#94119526, CASE WHEN ((numcos#94119377 = NA) OR (numcos#94119377 = null)) THEN null ELSE cast(numcos#94119377 as float) END AS numcos#94119528]\n+- FileScan csv [fractile#94119373,overall#94119374,annual_ret#94119375,annual_resret#94119376,numcos#94119377] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/transcripts/transcript..., PartitionFilters: []... [3614715] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614716 [labelType="html" label="MapPartitionsRDD [3614716] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614717 [labelType="html" label="MapPartitionsRDD [3614717] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614718 [labelType="html" label="MapPartitionsRDD [3614718] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285620 { label="WholeStageCodegen (1)"; 3614719 [labelType="html" label="MapPartitionsRDD [3614719] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285637 { label="Scan csv "; 3614720 [labelType="html" label="FileScanRDD [3614720] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614721 [labelType="html" label="MapPartitionsRDD [3614721] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285634 { label="WholeStageCodegen (1)"; 3614722 [labelType="html" label="MapPartitionsRDD [3614722] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285633 { label="InMemoryTableScan"; 3614723 [labelType="html" label="MapPartitionsRDD [3614723] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614724 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94119384 = NA) OR (fractile#94119384 = null)) THEN null ELSE cast(fractile#94119384 as int) END AS fractile#94119416, CASE WHEN ((cap#94119385 = NA) OR (cap#94119385 = null)) THEN null ELSE cast(cap#94119385 as float) END AS cap#94119418, CASE WHEN ((annual_ret#94119386 = NA) OR (annual_ret#94119386 = null)) THEN null ELSE cast(annual_ret#94119386 as float) END AS annual_ret#94119442, CASE WHEN ((annual_resret#94119387 = NA) OR (annual_resret#94119387 = null)) THEN null ELSE cast(annual_resret#94119387 as float) END AS annual_resret#94119465, CASE WHEN ((numcos#94119388 = NA) OR (numcos#94119388 = null)) THEN null ELSE cast(numcos#94119388 as float) END AS numcos#94119466]\n+- FileScan csv [fractile#94119384,cap#94119385,annual_ret#94119386,annual_resret#94119387,numcos#94119388] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/transcripts/transcript..., PartitionFilters: [], PushedFilters: [... [3614724] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614725 [labelType="html" label="MapPartitionsRDD [3614725] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614726 [labelType="html" label="MapPartitionsRDD [3614726] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614727 [labelType="html" label="MapPartitionsRDD [3614727] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285628 { label="WholeStageCodegen (2)"; 3614728 [labelType="html" label="MapPartitionsRDD [3614728] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285619 { label="Union"; 3614729 [labelType="html" label="UnionRDD [3614729] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285618 { label="InMemoryTableScan"; 3614730 [labelType="html" label="MapPartitionsRDD [3614730] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614731 [labelType="html" label="Union\n:- *(1) ColumnarToRow\n: +- InMemoryTableScan [fractile#94119415, annual_ret#94119441, annual_resret#94119526, numcos#94119528]\n: +- InMemoryRelation [fractile#94119415, overall#94119417, annual_ret#94119441, annual_resret#94119526, numcos#94119528], StorageLevel(disk, memory, deserialized, 1 replicas)\n: +- *(1) Project [CASE WHEN ((fractile#94119373 = NA) OR (fractile#94119373 = null)) THEN null ELSE cast(fractile#94119373 as int) END AS fractile#94119415, CASE WHEN ((overall#94119374 = NA) OR (overall#94119374 = null)) THEN null ELSE cast(overall#94119374 as int) END AS overall#94119417, CASE WHEN ((annual_ret#94119375 = NA) OR (annual_ret#94119375 = null)) THEN null ELSE cast(annual_ret#94119375 as float) END AS annual_ret#94119441, CASE WHEN ((annual_resret#94119376 = NA) OR (annual_resret#94119376 = null)) THEN null ELSE cast(annual_resret#94119376 as float) END AS annual_resret#94119526, CASE WHEN ((numcos#94119377 = NA) OR (numcos#94119377 = null)) THEN null ELSE cast(numco... [3614731] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614732 [labelType="html" label="MapPartitionsRDD [3614732] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614733 [labelType="html" label="MapPartitionsRDD [3614733] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614734 [labelType="html" label="MapPartitionsRDD [3614734] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285615 { label="WholeStageCodegen (1)"; 3614735 [labelType="html" label="MapPartitionsRDD [3614735] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285614 { label="CollectLimit"; 3614736 [labelType="html" label="MapPartitionsRDD [3614736] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614737 [labelType="html" label="MapPartitionsRDD [3614737] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3614711->3614712; 3614712->3614713; 3614713->3614714; 3614714->3614715; 3614715->3614716; 3614716->3614717; 3614717->3614718; 3614718->3614719; 3614720->3614721; 3614721->3614722; 3614722->3614723; 3614723->3614724; 3614724->3614725; 3614725->3614726; 3614726->3614727; 3614727->3614728; 3614719->3614729; 3614728->3614729; 3614729->3614730; 3614730->3614731; 3614731->3614732; 3614732->3614733; 3614733->3614734; 3614734->3614735; 3614735->3614736; 3614736->3614737; }

3614715

3614724

3614731

digraph G { subgraph clusterstage_511632 { label="Stage 511632"; subgraph cluster6285614 { label="CollectLimit"; 3614738 [labelType="html" label="ShuffledRowRDD [3614738] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614739 [labelType="html" label="MapPartitionsRDD [3614739] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285638 { label="mapPartitionsInternal"; 3614740 [labelType="html" label="MapPartitionsRDD [3614740] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285639 { label="map"; 3614741 [labelType="html" label="MapPartitionsRDD [3614741] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3614738->3614739; 3614739->3614740; 3614740->3614741; }

3614737,3614738

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
511632	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 15:59:44	9 ms	1/1			299.0 B
511631	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN ((fractile#94119373 = NA) OR (fractile#94119373 = null)) THEN null ELSE cast(fractile#94119373 as int) END AS fractile#94119415, CASE WHEN ((overall#94119374 = NA) OR (overall#94119374 = null)) THEN null ELSE cast(overall#94119374 as int) END AS overall#94119417, CASE WHEN ((annual_ret#94119375 = NA) OR (annual_ret#94119375 = null)) THEN null ELSE cast(annual_ret#94119375 as float) END AS annual_ret#94119441, CASE WHEN ((annual_resret#94119376 = NA) OR (annual_resret#94119376 = null)) THEN null ELSE cast(annual_resret#94119376 as float) END AS annual_resret#94119526, CASE WHEN ((numcos#94119377 = NA) OR (numcos#94119377 = null)) THEN null ELSE cast(numcos#94119377 as float) END AS numcos#94119528] +- FileScan csv [fractile#94119373,overall#94119374,annual_ret#94119375,annual_resret#94119376,numcos#94119377] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/transcripts/transcript..., PartitionFilters: []...(1) Project [CASE WHEN ((fractile#94119384 = NA) OR (fractile#94119384 = null)) THEN null ELSE cast(fractile#94119384 as int) END AS fractile#94119416, CASE WHEN ((cap#94119385 = NA) OR (cap#94119385 = null)) THEN null ELSE cast(cap#94119385 as float) END AS cap#94119418, CASE WHEN ((annual_ret#94119386 = NA) OR (annual_ret#94119386 = null)) THEN null ELSE cast(annual_ret#94119386 as float) END AS annual_ret#94119442, CASE WHEN ((annual_resret#94119387 = NA) OR (annual_resret#94119387 = null)) THEN null ELSE cast(annual_resret#94119387 as float) END AS annual_resret#94119465, CASE WHEN ((numcos#94119388 = NA) OR (numcos#94119388 = null)) THEN null ELSE cast(numcos#94119388 as float) END AS numcos#94119466] +- FileScan csv [fractile#94119384,cap#94119385,annual_ret#94119386,annual_resret#94119387,numcos#94119388] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/transcripts/transcript..., PartitionFilters: [], PushedFilters: [...Union :- (1) ColumnarToRow : +- InMemoryTableScan [fractile#94119415, annual_ret#94119441, annual_resret#94119526, numcos#94119528] : +- InMemoryRelation [fractile#94119415, overall#94119417, annual_ret#94119441, annual_resret#94119526, numcos#94119528], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN ((fractile#94119373 = NA) OR (fractile#94119373 = null)) THEN null ELSE cast(fractile#94119373 as int) END AS fractile#94119415, CASE WHEN ((overall#94119374 = NA) OR (overall#94119374 = null)) THEN null ELSE cast(overall#94119374 as int) END AS overall#94119417, CASE WHEN ((annual_ret#94119375 = NA) OR (annual_ret#94119375 = null)) THEN null ELSE cast(annual_ret#94119375 as float) END AS annual_ret#94119441, CASE WHEN ((annual_resret#94119376 = NA) OR (annual_resret#94119376 = null)) THEN null ELSE cast(annual_resret#94119376 as float) END AS annual_resret#94119526, CASE WHEN ((numcos#94119377 = NA) OR (numcos#94119377 = null)) THEN null ELSE cast(numco... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 15:59:44	53 ms	2/2	2.6 KiB			299.0 B

Page: