PlusAMP App Middleware - Details for Job 331439

Status: SUCCEEDED
Submitted: 2025/08/02 15:23:43
Duration: 58 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_513025 { label="Stage 513025"; subgraph cluster6302801 { label="Scan csv "; 3624562 [labelType="html" label="FileScanRDD [3624562] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624563 [labelType="html" label="MapPartitionsRDD [3624563] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302798 { label="WholeStageCodegen (1)"; 3624564 [labelType="html" label="MapPartitionsRDD [3624564] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302797 { label="InMemoryTableScan"; 3624565 [labelType="html" label="MapPartitionsRDD [3624565] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624566 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94376991 = NA) OR (fractile#94376991 = null)) THEN null ELSE cast(fractile#94376991 as float) END AS fractile#94377110, CASE WHEN ((overall#94376992 = NA) OR (overall#94376992 = null)) THEN null ELSE cast(overall#94376992 as int) END AS overall#94377111, CASE WHEN ((annual_ret#94376993 = NA) OR (annual_ret#94376993 = null)) THEN null ELSE cast(annual_ret#94376993 as float) END AS annual_ret#94377112, CASE WHEN ((annual_resret#94376994 = NA) OR (annual_resret#94376994 = null)) THEN null ELSE cast(annual_resret#94376994 as float) END AS annual_resret#94377114, CASE WHEN ((numcos#94376995 = NA) OR (numcos#94376995 = null)) THEN null ELSE cast(numcos#94376995 as float) END AS numcos#94377117]\n+- FileScan csv [fractile#94376991,overall#94376992,annual_ret#94376993,annual_resret#94376994,numcos#94376995] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: ... [3624566] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624567 [labelType="html" label="MapPartitionsRDD [3624567] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624568 [labelType="html" label="MapPartitionsRDD [3624568] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624569 [labelType="html" label="MapPartitionsRDD [3624569] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302794 { label="WholeStageCodegen (1)"; 3624570 [labelType="html" label="MapPartitionsRDD [3624570] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302811 { label="Scan csv "; 3624571 [labelType="html" label="FileScanRDD [3624571] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624572 [labelType="html" label="MapPartitionsRDD [3624572] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302808 { label="WholeStageCodegen (1)"; 3624573 [labelType="html" label="MapPartitionsRDD [3624573] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302807 { label="InMemoryTableScan"; 3624574 [labelType="html" label="MapPartitionsRDD [3624574] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624575 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94376996 = NA) OR (fractile#94376996 = null)) THEN null ELSE cast(fractile#94376996 as float) END AS fractile#94377216, CASE WHEN ((cap#94376997 = NA) OR (cap#94376997 = null)) THEN null ELSE cast(cap#94376997 as float) END AS cap#94377222, CASE WHEN ((annual_ret#94376998 = NA) OR (annual_ret#94376998 = null)) THEN null ELSE cast(annual_ret#94376998 as float) END AS annual_ret#94377224, CASE WHEN ((annual_resret#94376999 = NA) OR (annual_resret#94376999 = null)) THEN null ELSE cast(annual_resret#94376999 as float) END AS annual_resret#94377254, CASE WHEN ((numcos#94377000 = NA) OR (numcos#94377000 = null)) THEN null ELSE cast(numcos#94377000 as float) END AS numcos#94377280]\n+- FileScan csv [fractile#94376996,cap#94376997,annual_ret#94376998,annual_resret#94376999,numcos#94377000] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters:... [3624575] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624576 [labelType="html" label="MapPartitionsRDD [3624576] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624577 [labelType="html" label="MapPartitionsRDD [3624577] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624578 [labelType="html" label="MapPartitionsRDD [3624578] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302802 { label="WholeStageCodegen (2)"; 3624579 [labelType="html" label="MapPartitionsRDD [3624579] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302793 { label="Union"; 3624580 [labelType="html" label="UnionRDD [3624580] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302792 { label="InMemoryTableScan"; 3624581 [labelType="html" label="MapPartitionsRDD [3624581] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624582 [labelType="html" label="Union\n:- *(1) ColumnarToRow\n: +- InMemoryTableScan [fractile#94377110, annual_ret#94377112, annual_resret#94377114, numcos#94377117]\n: +- InMemoryRelation [fractile#94377110, overall#94377111, annual_ret#94377112, annual_resret#94377114, numcos#94377117], StorageLevel(disk, memory, deserialized, 1 replicas)\n: +- *(1) Project [CASE WHEN ((fractile#94376991 = NA) OR (fractile#94376991 = null)) THEN null ELSE cast(fractile#94376991 as float) END AS fractile#94377110, CASE WHEN ((overall#94376992 = NA) OR (overall#94376992 = null)) THEN null ELSE cast(overall#94376992 as int) END AS overall#94377111, CASE WHEN ((annual_ret#94376993 = NA) OR (annual_ret#94376993 = null)) THEN null ELSE cast(annual_ret#94376993 as float) END AS annual_ret#94377112, CASE WHEN ((annual_resret#94376994 = NA) OR (annual_resret#94376994 = null)) THEN null ELSE cast(annual_resret#94376994 as float) END AS annual_resret#94377114, CASE WHEN ((numcos#94376995 = NA) OR (numcos#94376995 = null)) THEN null ELSE cast(num... [3624582] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624583 [labelType="html" label="MapPartitionsRDD [3624583] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624584 [labelType="html" label="MapPartitionsRDD [3624584] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624585 [labelType="html" label="MapPartitionsRDD [3624585] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302789 { label="WholeStageCodegen (1)"; 3624586 [labelType="html" label="MapPartitionsRDD [3624586] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302788 { label="CollectLimit"; 3624587 [labelType="html" label="MapPartitionsRDD [3624587] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624588 [labelType="html" label="MapPartitionsRDD [3624588] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624562->3624563; 3624563->3624564; 3624564->3624565; 3624565->3624566; 3624566->3624567; 3624567->3624568; 3624568->3624569; 3624569->3624570; 3624571->3624572; 3624572->3624573; 3624573->3624574; 3624574->3624575; 3624575->3624576; 3624576->3624577; 3624577->3624578; 3624578->3624579; 3624570->3624580; 3624579->3624580; 3624580->3624581; 3624581->3624582; 3624582->3624583; 3624583->3624584; 3624584->3624585; 3624585->3624586; 3624586->3624587; 3624587->3624588; }

3624566

3624575

3624582

digraph G { subgraph clusterstage_513026 { label="Stage 513026"; subgraph cluster6302788 { label="CollectLimit"; 3624589 [labelType="html" label="ShuffledRowRDD [3624589] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624590 [labelType="html" label="MapPartitionsRDD [3624590] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302812 { label="mapPartitionsInternal"; 3624591 [labelType="html" label="MapPartitionsRDD [3624591] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302813 { label="map"; 3624592 [labelType="html" label="MapPartitionsRDD [3624592] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624589->3624590; 3624590->3624591; 3624591->3624592; }

3624588,3624589

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
513026	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:23:43	8 ms	1/1			288.0 B
513025	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN ((fractile#94376991 = NA) OR (fractile#94376991 = null)) THEN null ELSE cast(fractile#94376991 as float) END AS fractile#94377110, CASE WHEN ((overall#94376992 = NA) OR (overall#94376992 = null)) THEN null ELSE cast(overall#94376992 as int) END AS overall#94377111, CASE WHEN ((annual_ret#94376993 = NA) OR (annual_ret#94376993 = null)) THEN null ELSE cast(annual_ret#94376993 as float) END AS annual_ret#94377112, CASE WHEN ((annual_resret#94376994 = NA) OR (annual_resret#94376994 = null)) THEN null ELSE cast(annual_resret#94376994 as float) END AS annual_resret#94377114, CASE WHEN ((numcos#94376995 = NA) OR (numcos#94376995 = null)) THEN null ELSE cast(numcos#94376995 as float) END AS numcos#94377117] +- FileScan csv [fractile#94376991,overall#94376992,annual_ret#94376993,annual_resret#94376994,numcos#94376995] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: ...(1) Project [CASE WHEN ((fractile#94376996 = NA) OR (fractile#94376996 = null)) THEN null ELSE cast(fractile#94376996 as float) END AS fractile#94377216, CASE WHEN ((cap#94376997 = NA) OR (cap#94376997 = null)) THEN null ELSE cast(cap#94376997 as float) END AS cap#94377222, CASE WHEN ((annual_ret#94376998 = NA) OR (annual_ret#94376998 = null)) THEN null ELSE cast(annual_ret#94376998 as float) END AS annual_ret#94377224, CASE WHEN ((annual_resret#94376999 = NA) OR (annual_resret#94376999 = null)) THEN null ELSE cast(annual_resret#94376999 as float) END AS annual_resret#94377254, CASE WHEN ((numcos#94377000 = NA) OR (numcos#94377000 = null)) THEN null ELSE cast(numcos#94377000 as float) END AS numcos#94377280] +- FileScan csv [fractile#94376996,cap#94376997,annual_ret#94376998,annual_resret#94376999,numcos#94377000] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/leverage/..., PartitionFilters: [], PushedFilters:...Union :- (1) ColumnarToRow : +- InMemoryTableScan [fractile#94377110, annual_ret#94377112, annual_resret#94377114, numcos#94377117] : +- InMemoryRelation [fractile#94377110, overall#94377111, annual_ret#94377112, annual_resret#94377114, numcos#94377117], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN ((fractile#94376991 = NA) OR (fractile#94376991 = null)) THEN null ELSE cast(fractile#94376991 as float) END AS fractile#94377110, CASE WHEN ((overall#94376992 = NA) OR (overall#94376992 = null)) THEN null ELSE cast(overall#94376992 as int) END AS overall#94377111, CASE WHEN ((annual_ret#94376993 = NA) OR (annual_ret#94376993 = null)) THEN null ELSE cast(annual_ret#94376993 as float) END AS annual_ret#94377112, CASE WHEN ((annual_resret#94376994 = NA) OR (annual_resret#94376994 = null)) THEN null ELSE cast(annual_resret#94376994 as float) END AS annual_resret#94377114, CASE WHEN ((numcos#94376995 = NA) OR (numcos#94376995 = null)) THEN null ELSE cast(num... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:23:43	42 ms	2/2	2.7 KiB			288.0 B

Page: