PlusAMP App Middleware - Details for Job 331488

Status: SUCCEEDED
Submitted: 2025/08/02 15:25:27
Duration: 40 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_513099 { label="Stage 513099"; subgraph cluster6303734 { label="Scan csv "; 3625099 [labelType="html" label="FileScanRDD [3625099] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625100 [labelType="html" label="MapPartitionsRDD [3625100] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303731 { label="WholeStageCodegen (1)"; 3625101 [labelType="html" label="MapPartitionsRDD [3625101] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303730 { label="InMemoryTableScan"; 3625102 [labelType="html" label="MapPartitionsRDD [3625102] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625103 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94390930 = NA) OR (fractile#94390930 = null)) THEN null ELSE cast(fractile#94390930 as float) END AS fractile#94391017, CASE WHEN ((overall#94390931 = NA) OR (overall#94390931 = null)) THEN null ELSE cast(overall#94390931 as int) END AS overall#94391018, CASE WHEN ((annual_ret#94390932 = NA) OR (annual_ret#94390932 = null)) THEN null ELSE cast(annual_ret#94390932 as float) END AS annual_ret#94391024, CASE WHEN ((annual_resret#94390933 = NA) OR (annual_resret#94390933 = null)) THEN null ELSE cast(annual_resret#94390933 as float) END AS annual_resret#94391105, CASE WHEN ((numcos#94390934 = NA) OR (numcos#94390934 = null)) THEN null ELSE cast(numcos#94390934 as float) END AS numcos#94391106]\n+- FileScan csv [fractile#94390930,overall#94390931,annual_ret#94390932,annual_resret#94390933,numcos#94390934] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: ... [3625103] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625104 [labelType="html" label="MapPartitionsRDD [3625104] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625105 [labelType="html" label="MapPartitionsRDD [3625105] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625106 [labelType="html" label="MapPartitionsRDD [3625106] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303727 { label="WholeStageCodegen (1)"; 3625107 [labelType="html" label="MapPartitionsRDD [3625107] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303744 { label="Scan csv "; 3625108 [labelType="html" label="FileScanRDD [3625108] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625109 [labelType="html" label="MapPartitionsRDD [3625109] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303741 { label="WholeStageCodegen (1)"; 3625110 [labelType="html" label="MapPartitionsRDD [3625110] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303740 { label="InMemoryTableScan"; 3625111 [labelType="html" label="MapPartitionsRDD [3625111] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625112 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94390998 = NA) OR (fractile#94390998 = null)) THEN null ELSE cast(fractile#94390998 as float) END AS fractile#94391047, CASE WHEN ((cap#94390999 = NA) OR (cap#94390999 = null)) THEN null ELSE cast(cap#94390999 as float) END AS cap#94391048, CASE WHEN ((annual_ret#94391000 = NA) OR (annual_ret#94391000 = null)) THEN null ELSE cast(annual_ret#94391000 as float) END AS annual_ret#94391071, CASE WHEN ((annual_resret#94391001 = NA) OR (annual_resret#94391001 = null)) THEN null ELSE cast(annual_resret#94391001 as float) END AS annual_resret#94391072, CASE WHEN ((numcos#94391002 = NA) OR (numcos#94391002 = null)) THEN null ELSE cast(numcos#94391002 as float) END AS numcos#94391073]\n+- FileScan csv [fractile#94390998,cap#94390999,annual_ret#94391000,annual_resret#94391001,numcos#94391002] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters:... [3625112] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625113 [labelType="html" label="MapPartitionsRDD [3625113] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625114 [labelType="html" label="MapPartitionsRDD [3625114] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625115 [labelType="html" label="MapPartitionsRDD [3625115] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303735 { label="WholeStageCodegen (2)"; 3625116 [labelType="html" label="MapPartitionsRDD [3625116] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303726 { label="Union"; 3625117 [labelType="html" label="UnionRDD [3625117] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303725 { label="InMemoryTableScan"; 3625118 [labelType="html" label="MapPartitionsRDD [3625118] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625119 [labelType="html" label="Union\n:- *(1) ColumnarToRow\n: +- InMemoryTableScan [fractile#94391017, annual_ret#94391024, annual_resret#94391105, numcos#94391106]\n: +- InMemoryRelation [fractile#94391017, overall#94391018, annual_ret#94391024, annual_resret#94391105, numcos#94391106], StorageLevel(disk, memory, deserialized, 1 replicas)\n: +- *(1) Project [CASE WHEN ((fractile#94390930 = NA) OR (fractile#94390930 = null)) THEN null ELSE cast(fractile#94390930 as float) END AS fractile#94391017, CASE WHEN ((overall#94390931 = NA) OR (overall#94390931 = null)) THEN null ELSE cast(overall#94390931 as int) END AS overall#94391018, CASE WHEN ((annual_ret#94390932 = NA) OR (annual_ret#94390932 = null)) THEN null ELSE cast(annual_ret#94390932 as float) END AS annual_ret#94391024, CASE WHEN ((annual_resret#94390933 = NA) OR (annual_resret#94390933 = null)) THEN null ELSE cast(annual_resret#94390933 as float) END AS annual_resret#94391105, CASE WHEN ((numcos#94390934 = NA) OR (numcos#94390934 = null)) THEN null ELSE cast(num... [3625119] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625120 [labelType="html" label="MapPartitionsRDD [3625120] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625121 [labelType="html" label="MapPartitionsRDD [3625121] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625122 [labelType="html" label="MapPartitionsRDD [3625122] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303722 { label="WholeStageCodegen (1)"; 3625123 [labelType="html" label="MapPartitionsRDD [3625123] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303721 { label="CollectLimit"; 3625124 [labelType="html" label="MapPartitionsRDD [3625124] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625125 [labelType="html" label="MapPartitionsRDD [3625125] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625099->3625100; 3625100->3625101; 3625101->3625102; 3625102->3625103; 3625103->3625104; 3625104->3625105; 3625105->3625106; 3625106->3625107; 3625108->3625109; 3625109->3625110; 3625110->3625111; 3625111->3625112; 3625112->3625113; 3625113->3625114; 3625114->3625115; 3625115->3625116; 3625107->3625117; 3625116->3625117; 3625117->3625118; 3625118->3625119; 3625119->3625120; 3625120->3625121; 3625121->3625122; 3625122->3625123; 3625123->3625124; 3625124->3625125; }

3625103

3625112

3625119

digraph G { subgraph clusterstage_513100 { label="Stage 513100"; subgraph cluster6303721 { label="CollectLimit"; 3625126 [labelType="html" label="ShuffledRowRDD [3625126] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625127 [labelType="html" label="MapPartitionsRDD [3625127] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303745 { label="mapPartitionsInternal"; 3625128 [labelType="html" label="MapPartitionsRDD [3625128] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303746 { label="map"; 3625129 [labelType="html" label="MapPartitionsRDD [3625129] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625126->3625127; 3625127->3625128; 3625128->3625129; }

3625125,3625126

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
513100	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:25:27	7 ms	1/1			284.0 B
513099	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN ((fractile#94390930 = NA) OR (fractile#94390930 = null)) THEN null ELSE cast(fractile#94390930 as float) END AS fractile#94391017, CASE WHEN ((overall#94390931 = NA) OR (overall#94390931 = null)) THEN null ELSE cast(overall#94390931 as int) END AS overall#94391018, CASE WHEN ((annual_ret#94390932 = NA) OR (annual_ret#94390932 = null)) THEN null ELSE cast(annual_ret#94390932 as float) END AS annual_ret#94391024, CASE WHEN ((annual_resret#94390933 = NA) OR (annual_resret#94390933 = null)) THEN null ELSE cast(annual_resret#94390933 as float) END AS annual_resret#94391105, CASE WHEN ((numcos#94390934 = NA) OR (numcos#94390934 = null)) THEN null ELSE cast(numcos#94390934 as float) END AS numcos#94391106] +- FileScan csv [fractile#94390930,overall#94390931,annual_ret#94390932,annual_resret#94390933,numcos#94390934] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: ...(1) Project [CASE WHEN ((fractile#94390998 = NA) OR (fractile#94390998 = null)) THEN null ELSE cast(fractile#94390998 as float) END AS fractile#94391047, CASE WHEN ((cap#94390999 = NA) OR (cap#94390999 = null)) THEN null ELSE cast(cap#94390999 as float) END AS cap#94391048, CASE WHEN ((annual_ret#94391000 = NA) OR (annual_ret#94391000 = null)) THEN null ELSE cast(annual_ret#94391000 as float) END AS annual_ret#94391071, CASE WHEN ((annual_resret#94391001 = NA) OR (annual_resret#94391001 = null)) THEN null ELSE cast(annual_resret#94391001 as float) END AS annual_resret#94391072, CASE WHEN ((numcos#94391002 = NA) OR (numcos#94391002 = null)) THEN null ELSE cast(numcos#94391002 as float) END AS numcos#94391073] +- FileScan csv [fractile#94390998,cap#94390999,annual_ret#94391000,annual_resret#94391001,numcos#94391002] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/risk_factors/momentum/..., PartitionFilters: [], PushedFilters:...Union :- (1) ColumnarToRow : +- InMemoryTableScan [fractile#94391017, annual_ret#94391024, annual_resret#94391105, numcos#94391106] : +- InMemoryRelation [fractile#94391017, overall#94391018, annual_ret#94391024, annual_resret#94391105, numcos#94391106], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN ((fractile#94390930 = NA) OR (fractile#94390930 = null)) THEN null ELSE cast(fractile#94390930 as float) END AS fractile#94391017, CASE WHEN ((overall#94390931 = NA) OR (overall#94390931 = null)) THEN null ELSE cast(overall#94390931 as int) END AS overall#94391018, CASE WHEN ((annual_ret#94390932 = NA) OR (annual_ret#94390932 = null)) THEN null ELSE cast(annual_ret#94390932 as float) END AS annual_ret#94391024, CASE WHEN ((annual_resret#94390933 = NA) OR (annual_resret#94390933 = null)) THEN null ELSE cast(annual_resret#94390933 as float) END AS annual_resret#94391105, CASE WHEN ((numcos#94390934 = NA) OR (numcos#94390934 = null)) THEN null ELSE cast(num... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:25:27	27 ms	2/2	2.7 KiB			284.0 B

Page: