PlusAMP App Middleware - Details for Job 331341

Status: SUCCEEDED
Submitted: 2025/08/02 15:20:15
Duration: 47 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_512877 { label="Stage 512877"; subgraph cluster6300930 { label="Scan csv "; 3623482 [labelType="html" label="FileScanRDD [3623482] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623483 [labelType="html" label="MapPartitionsRDD [3623483] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300927 { label="WholeStageCodegen (1)"; 3623484 [labelType="html" label="MapPartitionsRDD [3623484] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300926 { label="InMemoryTableScan"; 3623485 [labelType="html" label="MapPartitionsRDD [3623485] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623486 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94348227 = NA) OR (fractile#94348227 = null)) THEN null ELSE cast(fractile#94348227 as float) END AS fractile#94348253, CASE WHEN ((overall#94348228 = NA) OR (overall#94348228 = null)) THEN null ELSE cast(overall#94348228 as int) END AS overall#94348254, CASE WHEN ((annual_ret#94348229 = NA) OR (annual_ret#94348229 = null)) THEN null ELSE cast(annual_ret#94348229 as float) END AS annual_ret#94348255, CASE WHEN ((annual_resret#94348230 = NA) OR (annual_resret#94348230 = null)) THEN null ELSE cast(annual_resret#94348230 as float) END AS annual_resret#94348256, CASE WHEN ((numcos#94348231 = NA) OR (numcos#94348231 = null)) THEN null ELSE cast(numcos#94348231 as float) END AS numcos#94348257]\n+- FileScan csv [fractile#94348227,overall#94348228,annual_ret#94348229,annual_resret#94348230,numcos#94348231] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: ... [3623486] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623487 [labelType="html" label="MapPartitionsRDD [3623487] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623488 [labelType="html" label="MapPartitionsRDD [3623488] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623489 [labelType="html" label="MapPartitionsRDD [3623489] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300923 { label="WholeStageCodegen (1)"; 3623490 [labelType="html" label="MapPartitionsRDD [3623490] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300940 { label="Scan csv "; 3623491 [labelType="html" label="FileScanRDD [3623491] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623492 [labelType="html" label="MapPartitionsRDD [3623492] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300937 { label="WholeStageCodegen (1)"; 3623493 [labelType="html" label="MapPartitionsRDD [3623493] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300936 { label="InMemoryTableScan"; 3623494 [labelType="html" label="MapPartitionsRDD [3623494] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623495 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94348206 = NA) OR (fractile#94348206 = null)) THEN null ELSE cast(fractile#94348206 as float) END AS fractile#94348232, CASE WHEN ((cap#94348207 = NA) OR (cap#94348207 = null)) THEN null ELSE cast(cap#94348207 as float) END AS cap#94348238, CASE WHEN ((annual_ret#94348208 = NA) OR (annual_ret#94348208 = null)) THEN null ELSE cast(annual_ret#94348208 as float) END AS annual_ret#94348239, CASE WHEN ((annual_resret#94348209 = NA) OR (annual_resret#94348209 = null)) THEN null ELSE cast(annual_resret#94348209 as float) END AS annual_resret#94348251, CASE WHEN ((numcos#94348210 = NA) OR (numcos#94348210 = null)) THEN null ELSE cast(numcos#94348210 as float) END AS numcos#94348252]\n+- FileScan csv [fractile#94348206,cap#94348207,annual_ret#94348208,annual_resret#94348209,numcos#94348210] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters:... [3623495] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623496 [labelType="html" label="MapPartitionsRDD [3623496] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623497 [labelType="html" label="MapPartitionsRDD [3623497] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623498 [labelType="html" label="MapPartitionsRDD [3623498] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300931 { label="WholeStageCodegen (2)"; 3623499 [labelType="html" label="MapPartitionsRDD [3623499] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300922 { label="Union"; 3623500 [labelType="html" label="UnionRDD [3623500] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300921 { label="InMemoryTableScan"; 3623501 [labelType="html" label="MapPartitionsRDD [3623501] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623502 [labelType="html" label="Union\n:- *(1) ColumnarToRow\n: +- InMemoryTableScan [fractile#94348253, annual_ret#94348255, annual_resret#94348256, numcos#94348257]\n: +- InMemoryRelation [fractile#94348253, overall#94348254, annual_ret#94348255, annual_resret#94348256, numcos#94348257], StorageLevel(disk, memory, deserialized, 1 replicas)\n: +- *(1) Project [CASE WHEN ((fractile#94348227 = NA) OR (fractile#94348227 = null)) THEN null ELSE cast(fractile#94348227 as float) END AS fractile#94348253, CASE WHEN ((overall#94348228 = NA) OR (overall#94348228 = null)) THEN null ELSE cast(overall#94348228 as int) END AS overall#94348254, CASE WHEN ((annual_ret#94348229 = NA) OR (annual_ret#94348229 = null)) THEN null ELSE cast(annual_ret#94348229 as float) END AS annual_ret#94348255, CASE WHEN ((annual_resret#94348230 = NA) OR (annual_resret#94348230 = null)) THEN null ELSE cast(annual_resret#94348230 as float) END AS annual_resret#94348256, CASE WHEN ((numcos#94348231 = NA) OR (numcos#94348231 = null)) THEN null ELSE cast(num... [3623502] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623503 [labelType="html" label="MapPartitionsRDD [3623503] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623504 [labelType="html" label="MapPartitionsRDD [3623504] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623505 [labelType="html" label="MapPartitionsRDD [3623505] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300918 { label="WholeStageCodegen (1)"; 3623506 [labelType="html" label="MapPartitionsRDD [3623506] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300917 { label="CollectLimit"; 3623507 [labelType="html" label="MapPartitionsRDD [3623507] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623508 [labelType="html" label="MapPartitionsRDD [3623508] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623482->3623483; 3623483->3623484; 3623484->3623485; 3623485->3623486; 3623486->3623487; 3623487->3623488; 3623488->3623489; 3623489->3623490; 3623491->3623492; 3623492->3623493; 3623493->3623494; 3623494->3623495; 3623495->3623496; 3623496->3623497; 3623497->3623498; 3623498->3623499; 3623490->3623500; 3623499->3623500; 3623500->3623501; 3623501->3623502; 3623502->3623503; 3623503->3623504; 3623504->3623505; 3623505->3623506; 3623506->3623507; 3623507->3623508; }

3623486

3623495

3623502

digraph G { subgraph clusterstage_512878 { label="Stage 512878"; subgraph cluster6300917 { label="CollectLimit"; 3623509 [labelType="html" label="ShuffledRowRDD [3623509] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623510 [labelType="html" label="MapPartitionsRDD [3623510] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300941 { label="mapPartitionsInternal"; 3623511 [labelType="html" label="MapPartitionsRDD [3623511] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300942 { label="map"; 3623512 [labelType="html" label="MapPartitionsRDD [3623512] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623509->3623510; 3623510->3623511; 3623511->3623512; }

3623508,3623509

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
512878	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:15	7 ms	1/1			281.0 B
512877	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN ((fractile#94348227 = NA) OR (fractile#94348227 = null)) THEN null ELSE cast(fractile#94348227 as float) END AS fractile#94348253, CASE WHEN ((overall#94348228 = NA) OR (overall#94348228 = null)) THEN null ELSE cast(overall#94348228 as int) END AS overall#94348254, CASE WHEN ((annual_ret#94348229 = NA) OR (annual_ret#94348229 = null)) THEN null ELSE cast(annual_ret#94348229 as float) END AS annual_ret#94348255, CASE WHEN ((annual_resret#94348230 = NA) OR (annual_resret#94348230 = null)) THEN null ELSE cast(annual_resret#94348230 as float) END AS annual_resret#94348256, CASE WHEN ((numcos#94348231 = NA) OR (numcos#94348231 = null)) THEN null ELSE cast(numcos#94348231 as float) END AS numcos#94348257] +- FileScan csv [fractile#94348227,overall#94348228,annual_ret#94348229,annual_resret#94348230,numcos#94348231] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: ...(1) Project [CASE WHEN ((fractile#94348206 = NA) OR (fractile#94348206 = null)) THEN null ELSE cast(fractile#94348206 as float) END AS fractile#94348232, CASE WHEN ((cap#94348207 = NA) OR (cap#94348207 = null)) THEN null ELSE cast(cap#94348207 as float) END AS cap#94348238, CASE WHEN ((annual_ret#94348208 = NA) OR (annual_ret#94348208 = null)) THEN null ELSE cast(annual_ret#94348208 as float) END AS annual_ret#94348239, CASE WHEN ((annual_resret#94348209 = NA) OR (annual_resret#94348209 = null)) THEN null ELSE cast(annual_resret#94348209 as float) END AS annual_resret#94348251, CASE WHEN ((numcos#94348210 = NA) OR (numcos#94348210 = null)) THEN null ELSE cast(numcos#94348210 as float) END AS numcos#94348252] +- FileScan csv [fractile#94348206,cap#94348207,annual_ret#94348208,annual_resret#94348209,numcos#94348210] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters:...Union :- (1) ColumnarToRow : +- InMemoryTableScan [fractile#94348253, annual_ret#94348255, annual_resret#94348256, numcos#94348257] : +- InMemoryRelation [fractile#94348253, overall#94348254, annual_ret#94348255, annual_resret#94348256, numcos#94348257], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN ((fractile#94348227 = NA) OR (fractile#94348227 = null)) THEN null ELSE cast(fractile#94348227 as float) END AS fractile#94348253, CASE WHEN ((overall#94348228 = NA) OR (overall#94348228 = null)) THEN null ELSE cast(overall#94348228 as int) END AS overall#94348254, CASE WHEN ((annual_ret#94348229 = NA) OR (annual_ret#94348229 = null)) THEN null ELSE cast(annual_ret#94348229 as float) END AS annual_ret#94348255, CASE WHEN ((annual_resret#94348230 = NA) OR (annual_resret#94348230 = null)) THEN null ELSE cast(annual_resret#94348230 as float) END AS annual_resret#94348256, CASE WHEN ((numcos#94348231 = NA) OR (numcos#94348231 = null)) THEN null ELSE cast(num... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:20:15	34 ms	2/2	2.7 KiB			281.0 B

Page: