PlusAMP App Middleware - Details for Job 330929

Status: SUCCEEDED
Submitted: 2025/08/02 15:06:25
Duration: 1 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512253 { label="Stage 512253"; subgraph cluster6292952 { label="Scan csv "; 3618911 [labelType="html" label="FileScanRDD [3618911] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618912 [labelType="html" label="MapPartitionsRDD [3618912] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292949 { label="WholeStageCodegen (1)"; 3618913 [labelType="html" label="MapPartitionsRDD [3618913] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292948 { label="InMemoryTableScan"; 3618914 [labelType="html" label="MapPartitionsRDD [3618914] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618915 [labelType="html" label="*(1) Project [CASE WHEN (date#94230794 = null) THEN null ELSE cast(date#94230794 as date) END AS date#94231100, CASE WHEN ((overall#94230795 = NA) OR (overall#94230795 = null)) THEN null ELSE cast(overall#94230795 as int) END AS overall#94231103, CASE WHEN ((ret#94230796 = NA) OR (ret#94230796 = null)) THEN null ELSE cast(ret#94230796 as float) END AS ret#94231107, CASE WHEN ((resret#94230797 = NA) OR (resret#94230797 = null)) THEN null ELSE cast(resret#94230797 as float) END AS resret#94231110, CASE WHEN ((retnet#94230798 = NA) OR (retnet#94230798 = null)) THEN null ELSE cast(retnet#94230798 as float) END AS retnet#94231115, CASE WHEN ((turnover#94230799 = NA) OR (turnover#94230799 = null)) THEN null ELSE cast(turnover#94230799 as float) END AS turnover#94231119, CASE WHEN ((numcos#94230800 = NA) OR (numcos#94230800 = null)) THEN null ELSE cast(numcos#94230800 as float) END AS numcos#94231122, CASE WHEN ((benchmark#94230801 = NA) OR (benchmark#94230801 = null)) THEN null ELSE cast(benchmark#94230801 as fl... [3618915] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618916 [labelType="html" label="MapPartitionsRDD [3618916] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618917 [labelType="html" label="MapPartitionsRDD [3618917] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618918 [labelType="html" label="MapPartitionsRDD [3618918] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292961 { label="Scan csv "; 3618919 [labelType="html" label="FileScanRDD [3618919] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618920 [labelType="html" label="MapPartitionsRDD [3618920] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292958 { label="WholeStageCodegen (1)"; 3618921 [labelType="html" label="MapPartitionsRDD [3618921] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292957 { label="InMemoryTableScan"; 3618922 [labelType="html" label="MapPartitionsRDD [3618922] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618923 [labelType="html" label="*(1) Project [CASE WHEN (date#94230816 = null) THEN null ELSE cast(date#94230816 as date) END AS date#94231101, CASE WHEN ((cap#94230817 = NA) OR (cap#94230817 = null)) THEN null ELSE cast(cap#94230817 as float) END AS cap#94231104, CASE WHEN ((ret#94230818 = NA) OR (ret#94230818 = null)) THEN null ELSE cast(ret#94230818 as float) END AS ret#94231108, CASE WHEN ((resret#94230819 = NA) OR (resret#94230819 = null)) THEN null ELSE cast(resret#94230819 as float) END AS resret#94231112, CASE WHEN ((retnet#94230820 = NA) OR (retnet#94230820 = null)) THEN null ELSE cast(retnet#94230820 as float) END AS retnet#94231116, CASE WHEN ((turnover#94230821 = NA) OR (turnover#94230821 = null)) THEN null ELSE cast(turnover#94230821 as float) END AS turnover#94231120, CASE WHEN ((numcos#94230822 = NA) OR (numcos#94230822 = null)) THEN null ELSE cast(numcos#94230822 as float) END AS numcos#94231127, CASE WHEN ((coverage#94230823 = NA) OR (coverage#94230823 = null)) THEN null ELSE cast(coverage#94230823 as float) END AS cover... [3618923] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618924 [labelType="html" label="MapPartitionsRDD [3618924] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618925 [labelType="html" label="MapPartitionsRDD [3618925] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618926 [labelType="html" label="MapPartitionsRDD [3618926] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292953 { label="WholeStageCodegen (1)"; 3618927 [labelType="html" label="MapPartitionsRDD [3618927] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292947 { label="Union"; 3618928 [labelType="html" label="UnionRDD [3618928] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292946 { label="Exchange"; 3618953 [labelType="html" label="MapPartitionsRDD [3618953] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3618911->3618912; 3618912->3618913; 3618913->3618914; 3618914->3618915; 3618915->3618916; 3618916->3618917; 3618917->3618918; 3618919->3618920; 3618920->3618921; 3618921->3618922; 3618922->3618923; 3618923->3618924; 3618924->3618925; 3618925->3618926; 3618926->3618927; 3618918->3618928; 3618927->3618928; 3618928->3618953; }

3618915

3618923

digraph G { subgraph clusterstage_512254 { label="Stage 512254"; subgraph cluster6292946 { label="Exchange"; 3618954 [labelType="html" label="ShuffledRowRDD [3618954] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292943 { label="WholeStageCodegen (2)"; 3618955 [labelType="html" label="MapPartitionsRDD [3618955] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292942 { label="InMemoryTableScan"; 3618956 [labelType="html" label="MapPartitionsRDD [3618956] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618957 [labelType="html" label="*(2) Sort [date#94231100 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94231100 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7523548]\n +- Union\n :- InMemoryTableScan [date#94231100, numcos#94231122]\n : +- InMemoryRelation [date#94231100, overall#94231103, ret#94231107, resret#94231110, retnet#94231115, turnover#94231119, numcos#94231122, benchmark#94231125, excess_ret#94231131, excess_resret#94231135, excess_retnet#94231140], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94230794 = null) THEN null ELSE cast(date#94230794 as date) END AS date#94231100, CASE WHEN ((overall#94230795 = NA) OR (overall#94230795 = null)) THEN null ELSE cast(overall#94230795 as int) END AS overall#94231103, CASE WHEN ((ret#94230796 = NA) OR (ret#94230796 = null)) THEN null ELSE cast(ret#94230796 as float) END AS ret#94231107, CASE WHEN ((resret#94230797 = NA) OR (resret#94230797 = null)) THEN null ELSE cast(resret#94230797 as float) END A... [3618957] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618958 [labelType="html" label="MapPartitionsRDD [3618958] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618959 [labelType="html" label="MapPartitionsRDD [3618959] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618960 [labelType="html" label="MapPartitionsRDD [3618960] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6292941 { label="CollectLimit"; 3618961 [labelType="html" label="MapPartitionsRDD [3618961] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618962 [labelType="html" label="MapPartitionsRDD [3618962] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3618954->3618955; 3618955->3618956; 3618956->3618957; 3618957->3618958; 3618958->3618959; 3618959->3618960; 3618960->3618961; 3618961->3618962; }

3618953,3618954

3618957

digraph G { subgraph clusterstage_512255 { label="Stage 512255"; subgraph cluster6292941 { label="CollectLimit"; 3618963 [labelType="html" label="ShuffledRowRDD [3618963] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3618964 [labelType="html" label="MapPartitionsRDD [3618964] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293004 { label="mapPartitionsInternal"; 3618965 [labelType="html" label="MapPartitionsRDD [3618965] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6293005 { label="map"; 3618966 [labelType="html" label="MapPartitionsRDD [3618966] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3618963->3618964; 3618964->3618965; 3618965->3618966; }

3618962,3618963

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512255	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:06:27	19 ms	1/1		45.9 KiB
512254	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94231100 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94231100 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7523548] +- Union :- InMemoryTableScan [date#94231100, numcos#94231122] : +- InMemoryRelation [date#94231100, overall#94231103, ret#94231107, resret#94231110, retnet#94231115, turnover#94231119, numcos#94231122, benchmark#94231125, excess_ret#94231131, excess_resret#94231135, excess_retnet#94231140], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94230794 = null) THEN null ELSE cast(date#94230794 as date) END AS date#94231100, CASE WHEN ((overall#94230795 = NA) OR (overall#94230795 = null)) THEN null ELSE cast(overall#94230795 as int) END AS overall#94231103, CASE WHEN ((ret#94230796 = NA) OR (ret#94230796 = null)) THEN null ELSE cast(ret#94230796 as float) END AS ret#94231107, CASE WHEN ((resret#94230797 = NA) OR (resret#94230797 = null)) THEN null ELSE cast(resret#94230797 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:06:26	0.7 s	200/200		45.9 KiB	45.9 KiB
512253	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94230794 = null) THEN null ELSE cast(date#94230794 as date) END AS date#94231100, CASE WHEN ((overall#94230795 = NA) OR (overall#94230795 = null)) THEN null ELSE cast(overall#94230795 as int) END AS overall#94231103, CASE WHEN ((ret#94230796 = NA) OR (ret#94230796 = null)) THEN null ELSE cast(ret#94230796 as float) END AS ret#94231107, CASE WHEN ((resret#94230797 = NA) OR (resret#94230797 = null)) THEN null ELSE cast(resret#94230797 as float) END AS resret#94231110, CASE WHEN ((retnet#94230798 = NA) OR (retnet#94230798 = null)) THEN null ELSE cast(retnet#94230798 as float) END AS retnet#94231115, CASE WHEN ((turnover#94230799 = NA) OR (turnover#94230799 = null)) THEN null ELSE cast(turnover#94230799 as float) END AS turnover#94231119, CASE WHEN ((numcos#94230800 = NA) OR (numcos#94230800 = null)) THEN null ELSE cast(numcos#94230800 as float) END AS numcos#94231122, CASE WHEN ((benchmark#94230801 = NA) OR (benchmark#94230801 = null)) THEN null ELSE cast(benchmark#94230801 as fl...(1) Project [CASE WHEN (date#94230816 = null) THEN null ELSE cast(date#94230816 as date) END AS date#94231101, CASE WHEN ((cap#94230817 = NA) OR (cap#94230817 = null)) THEN null ELSE cast(cap#94230817 as float) END AS cap#94231104, CASE WHEN ((ret#94230818 = NA) OR (ret#94230818 = null)) THEN null ELSE cast(ret#94230818 as float) END AS ret#94231108, CASE WHEN ((resret#94230819 = NA) OR (resret#94230819 = null)) THEN null ELSE cast(resret#94230819 as float) END AS resret#94231112, CASE WHEN ((retnet#94230820 = NA) OR (retnet#94230820 = null)) THEN null ELSE cast(retnet#94230820 as float) END AS retnet#94231116, CASE WHEN ((turnover#94230821 = NA) OR (turnover#94230821 = null)) THEN null ELSE cast(turnover#94230821 as float) END AS turnover#94231120, CASE WHEN ((numcos#94230822 = NA) OR (numcos#94230822 = null)) THEN null ELSE cast(numcos#94230822 as float) END AS numcos#94231127, CASE WHEN ((coverage#94230823 = NA) OR (coverage#94230823 = null)) THEN null ELSE cast(coverage#94230823 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:06:25	50 ms	2/2	819.1 KiB		45.9 KiB

Page: