PlusAMP App Middleware - Details for Job 331292

Status: SUCCEEDED
Submitted: 2025/08/02 15:18:33
Duration: 46 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_512803 { label="Stage 512803"; subgraph cluster6299986 { label="Scan csv "; 3622939 [labelType="html" label="FileScanRDD [3622939] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622940 [labelType="html" label="MapPartitionsRDD [3622940] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299983 { label="WholeStageCodegen (1)"; 3622941 [labelType="html" label="MapPartitionsRDD [3622941] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299982 { label="InMemoryTableScan"; 3622942 [labelType="html" label="MapPartitionsRDD [3622942] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622943 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94333959 = NA) OR (fractile#94333959 = null)) THEN null ELSE cast(fractile#94333959 as float) END AS fractile#94334079, CASE WHEN ((overall#94333960 = NA) OR (overall#94333960 = null)) THEN null ELSE cast(overall#94333960 as int) END AS overall#94334106, CASE WHEN ((annual_ret#94333961 = NA) OR (annual_ret#94333961 = null)) THEN null ELSE cast(annual_ret#94333961 as float) END AS annual_ret#94334107, CASE WHEN ((annual_resret#94333962 = NA) OR (annual_resret#94333962 = null)) THEN null ELSE cast(annual_resret#94333962 as float) END AS annual_resret#94334108, CASE WHEN ((numcos#94333963 = NA) OR (numcos#94333963 = null)) THEN null ELSE cast(numcos#94333963 as float) END AS numcos#94334109]\n+- FileScan csv [fractile#94333959,overall#94333960,annual_ret#94333961,annual_resret#94333962,numcos#94333963] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: ... [3622943] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622944 [labelType="html" label="MapPartitionsRDD [3622944] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622945 [labelType="html" label="MapPartitionsRDD [3622945] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622946 [labelType="html" label="MapPartitionsRDD [3622946] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299979 { label="WholeStageCodegen (1)"; 3622947 [labelType="html" label="MapPartitionsRDD [3622947] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299996 { label="Scan csv "; 3622948 [labelType="html" label="FileScanRDD [3622948] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622949 [labelType="html" label="MapPartitionsRDD [3622949] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299993 { label="WholeStageCodegen (1)"; 3622950 [labelType="html" label="MapPartitionsRDD [3622950] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299992 { label="InMemoryTableScan"; 3622951 [labelType="html" label="MapPartitionsRDD [3622951] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622952 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94333965 = NA) OR (fractile#94333965 = null)) THEN null ELSE cast(fractile#94333965 as float) END AS fractile#94334080, CASE WHEN ((cap#94333966 = NA) OR (cap#94333966 = null)) THEN null ELSE cast(cap#94333966 as float) END AS cap#94334090, CASE WHEN ((annual_ret#94333967 = NA) OR (annual_ret#94333967 = null)) THEN null ELSE cast(annual_ret#94333967 as float) END AS annual_ret#94334104, CASE WHEN ((annual_resret#94333968 = NA) OR (annual_resret#94333968 = null)) THEN null ELSE cast(annual_resret#94333968 as float) END AS annual_resret#94334105, CASE WHEN ((numcos#94333969 = NA) OR (numcos#94333969 = null)) THEN null ELSE cast(numcos#94333969 as float) END AS numcos#94334128]\n+- FileScan csv [fractile#94333965,cap#94333966,annual_ret#94333967,annual_resret#94333968,numcos#94333969] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters:... [3622952] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622953 [labelType="html" label="MapPartitionsRDD [3622953] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622954 [labelType="html" label="MapPartitionsRDD [3622954] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622955 [labelType="html" label="MapPartitionsRDD [3622955] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299987 { label="WholeStageCodegen (2)"; 3622956 [labelType="html" label="MapPartitionsRDD [3622956] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299978 { label="Union"; 3622957 [labelType="html" label="UnionRDD [3622957] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299977 { label="InMemoryTableScan"; 3622958 [labelType="html" label="MapPartitionsRDD [3622958] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622959 [labelType="html" label="Union\n:- *(1) ColumnarToRow\n: +- InMemoryTableScan [fractile#94334079, annual_ret#94334107, annual_resret#94334108, numcos#94334109]\n: +- InMemoryRelation [fractile#94334079, overall#94334106, annual_ret#94334107, annual_resret#94334108, numcos#94334109], StorageLevel(disk, memory, deserialized, 1 replicas)\n: +- *(1) Project [CASE WHEN ((fractile#94333959 = NA) OR (fractile#94333959 = null)) THEN null ELSE cast(fractile#94333959 as float) END AS fractile#94334079, CASE WHEN ((overall#94333960 = NA) OR (overall#94333960 = null)) THEN null ELSE cast(overall#94333960 as int) END AS overall#94334106, CASE WHEN ((annual_ret#94333961 = NA) OR (annual_ret#94333961 = null)) THEN null ELSE cast(annual_ret#94333961 as float) END AS annual_ret#94334107, CASE WHEN ((annual_resret#94333962 = NA) OR (annual_resret#94333962 = null)) THEN null ELSE cast(annual_resret#94333962 as float) END AS annual_resret#94334108, CASE WHEN ((numcos#94333963 = NA) OR (numcos#94333963 = null)) THEN null ELSE cast(num... [3622959] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622960 [labelType="html" label="MapPartitionsRDD [3622960] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622961 [labelType="html" label="MapPartitionsRDD [3622961] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622962 [labelType="html" label="MapPartitionsRDD [3622962] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299974 { label="WholeStageCodegen (1)"; 3622963 [labelType="html" label="MapPartitionsRDD [3622963] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299973 { label="CollectLimit"; 3622964 [labelType="html" label="MapPartitionsRDD [3622964] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622965 [labelType="html" label="MapPartitionsRDD [3622965] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622939->3622940; 3622940->3622941; 3622941->3622942; 3622942->3622943; 3622943->3622944; 3622944->3622945; 3622945->3622946; 3622946->3622947; 3622948->3622949; 3622949->3622950; 3622950->3622951; 3622951->3622952; 3622952->3622953; 3622953->3622954; 3622954->3622955; 3622955->3622956; 3622947->3622957; 3622956->3622957; 3622957->3622958; 3622958->3622959; 3622959->3622960; 3622960->3622961; 3622961->3622962; 3622962->3622963; 3622963->3622964; 3622964->3622965; }

3622943

3622952

3622959

digraph G { subgraph clusterstage_512804 { label="Stage 512804"; subgraph cluster6299973 { label="CollectLimit"; 3622966 [labelType="html" label="ShuffledRowRDD [3622966] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3622967 [labelType="html" label="MapPartitionsRDD [3622967] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299997 { label="mapPartitionsInternal"; 3622968 [labelType="html" label="MapPartitionsRDD [3622968] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6299998 { label="map"; 3622969 [labelType="html" label="MapPartitionsRDD [3622969] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3622966->3622967; 3622967->3622968; 3622968->3622969; }

3622965,3622966

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
512804	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:18:33	8 ms	1/1			286.0 B
512803	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN ((fractile#94333959 = NA) OR (fractile#94333959 = null)) THEN null ELSE cast(fractile#94333959 as float) END AS fractile#94334079, CASE WHEN ((overall#94333960 = NA) OR (overall#94333960 = null)) THEN null ELSE cast(overall#94333960 as int) END AS overall#94334106, CASE WHEN ((annual_ret#94333961 = NA) OR (annual_ret#94333961 = null)) THEN null ELSE cast(annual_ret#94333961 as float) END AS annual_ret#94334107, CASE WHEN ((annual_resret#94333962 = NA) OR (annual_resret#94333962 = null)) THEN null ELSE cast(annual_resret#94333962 as float) END AS annual_resret#94334108, CASE WHEN ((numcos#94333963 = NA) OR (numcos#94333963 = null)) THEN null ELSE cast(numcos#94333963 as float) END AS numcos#94334109] +- FileScan csv [fractile#94333959,overall#94333960,annual_ret#94333961,annual_resret#94333962,numcos#94333963] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: ...(1) Project [CASE WHEN ((fractile#94333965 = NA) OR (fractile#94333965 = null)) THEN null ELSE cast(fractile#94333965 as float) END AS fractile#94334080, CASE WHEN ((cap#94333966 = NA) OR (cap#94333966 = null)) THEN null ELSE cast(cap#94333966 as float) END AS cap#94334090, CASE WHEN ((annual_ret#94333967 = NA) OR (annual_ret#94333967 = null)) THEN null ELSE cast(annual_ret#94333967 as float) END AS annual_ret#94334104, CASE WHEN ((annual_resret#94333968 = NA) OR (annual_resret#94333968 = null)) THEN null ELSE cast(annual_resret#94333968 as float) END AS annual_resret#94334105, CASE WHEN ((numcos#94333969 = NA) OR (numcos#94333969 = null)) THEN null ELSE cast(numcos#94333969 as float) END AS numcos#94334128] +- FileScan csv [fractile#94333965,cap#94333966,annual_ret#94333967,annual_resret#94333968,numcos#94333969] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/estimize_signal_histor..., PartitionFilters: [], PushedFilters:...Union :- (1) ColumnarToRow : +- InMemoryTableScan [fractile#94334079, annual_ret#94334107, annual_resret#94334108, numcos#94334109] : +- InMemoryRelation [fractile#94334079, overall#94334106, annual_ret#94334107, annual_resret#94334108, numcos#94334109], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN ((fractile#94333959 = NA) OR (fractile#94333959 = null)) THEN null ELSE cast(fractile#94333959 as float) END AS fractile#94334079, CASE WHEN ((overall#94333960 = NA) OR (overall#94333960 = null)) THEN null ELSE cast(overall#94333960 as int) END AS overall#94334106, CASE WHEN ((annual_ret#94333961 = NA) OR (annual_ret#94333961 = null)) THEN null ELSE cast(annual_ret#94333961 as float) END AS annual_ret#94334107, CASE WHEN ((annual_resret#94333962 = NA) OR (annual_resret#94333962 = null)) THEN null ELSE cast(annual_resret#94333962 as float) END AS annual_resret#94334108, CASE WHEN ((numcos#94333963 = NA) OR (numcos#94333963 = null)) THEN null ELSE cast(num... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:18:33	32 ms	2/2	2.7 KiB			286.0 B

Page: