PlusAMP App Middleware - Details for Job 331110

Status: SUCCEEDED
Submitted: 2025/08/02 15:12:06
Duration: 0.9 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512526 { label="Stage 512526"; subgraph cluster6296492 { label="Scan csv "; 3620928 [labelType="html" label="FileScanRDD [3620928] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620929 [labelType="html" label="MapPartitionsRDD [3620929] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296489 { label="WholeStageCodegen (1)"; 3620930 [labelType="html" label="MapPartitionsRDD [3620930] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296488 { label="InMemoryTableScan"; 3620931 [labelType="html" label="MapPartitionsRDD [3620931] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620932 [labelType="html" label="*(1) Project [CASE WHEN (date#94282340 = null) THEN null ELSE cast(date#94282340 as date) END AS date#94282656, CASE WHEN ((overall#94282341 = NA) OR (overall#94282341 = null)) THEN null ELSE cast(overall#94282341 as int) END AS overall#94282662, CASE WHEN ((ret#94282342 = NA) OR (ret#94282342 = null)) THEN null ELSE cast(ret#94282342 as float) END AS ret#94282666, CASE WHEN ((resret#94282343 = NA) OR (resret#94282343 = null)) THEN null ELSE cast(resret#94282343 as float) END AS resret#94282668, CASE WHEN ((retnet#94282344 = NA) OR (retnet#94282344 = null)) THEN null ELSE cast(retnet#94282344 as float) END AS retnet#94282671, CASE WHEN ((turnover#94282345 = NA) OR (turnover#94282345 = null)) THEN null ELSE cast(turnover#94282345 as float) END AS turnover#94282675, CASE WHEN ((numcos#94282346 = NA) OR (numcos#94282346 = null)) THEN null ELSE cast(numcos#94282346 as float) END AS numcos#94282679, CASE WHEN ((benchmark#94282347 = NA) OR (benchmark#94282347 = null)) THEN null ELSE cast(benchmark#94282347 as fl... [3620932] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620933 [labelType="html" label="MapPartitionsRDD [3620933] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620934 [labelType="html" label="MapPartitionsRDD [3620934] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620935 [labelType="html" label="MapPartitionsRDD [3620935] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296501 { label="Scan csv "; 3620936 [labelType="html" label="FileScanRDD [3620936] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620937 [labelType="html" label="MapPartitionsRDD [3620937] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296498 { label="WholeStageCodegen (1)"; 3620938 [labelType="html" label="MapPartitionsRDD [3620938] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296497 { label="InMemoryTableScan"; 3620939 [labelType="html" label="MapPartitionsRDD [3620939] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620940 [labelType="html" label="*(1) Project [CASE WHEN (date#94282421 = null) THEN null ELSE cast(date#94282421 as date) END AS date#94282548, CASE WHEN ((cap#94282422 = NA) OR (cap#94282422 = null)) THEN null ELSE cast(cap#94282422 as float) END AS cap#94282549, CASE WHEN ((ret#94282423 = NA) OR (ret#94282423 = null)) THEN null ELSE cast(ret#94282423 as float) END AS ret#94282602, CASE WHEN ((resret#94282424 = NA) OR (resret#94282424 = null)) THEN null ELSE cast(resret#94282424 as float) END AS resret#94282603, CASE WHEN ((retnet#94282425 = NA) OR (retnet#94282425 = null)) THEN null ELSE cast(retnet#94282425 as float) END AS retnet#94282615, CASE WHEN ((turnover#94282426 = NA) OR (turnover#94282426 = null)) THEN null ELSE cast(turnover#94282426 as float) END AS turnover#94282653, CASE WHEN ((numcos#94282427 = NA) OR (numcos#94282427 = null)) THEN null ELSE cast(numcos#94282427 as float) END AS numcos#94282655, CASE WHEN ((coverage#94282428 = NA) OR (coverage#94282428 = null)) THEN null ELSE cast(coverage#94282428 as float) END AS cover... [3620940] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620941 [labelType="html" label="MapPartitionsRDD [3620941] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620942 [labelType="html" label="MapPartitionsRDD [3620942] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620943 [labelType="html" label="MapPartitionsRDD [3620943] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296493 { label="WholeStageCodegen (1)"; 3620944 [labelType="html" label="MapPartitionsRDD [3620944] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296487 { label="Union"; 3620945 [labelType="html" label="UnionRDD [3620945] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296486 { label="Exchange"; 3620949 [labelType="html" label="MapPartitionsRDD [3620949] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3620928->3620929; 3620929->3620930; 3620930->3620931; 3620931->3620932; 3620932->3620933; 3620933->3620934; 3620934->3620935; 3620936->3620937; 3620937->3620938; 3620938->3620939; 3620939->3620940; 3620940->3620941; 3620941->3620942; 3620942->3620943; 3620943->3620944; 3620935->3620945; 3620944->3620945; 3620945->3620949; }

3620932

3620940

digraph G { subgraph clusterstage_512527 { label="Stage 512527"; subgraph cluster6296486 { label="Exchange"; 3620950 [labelType="html" label="ShuffledRowRDD [3620950] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296483 { label="WholeStageCodegen (2)"; 3620951 [labelType="html" label="MapPartitionsRDD [3620951] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296482 { label="InMemoryTableScan"; 3620952 [labelType="html" label="MapPartitionsRDD [3620952] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620953 [labelType="html" label="*(2) Sort [date#94282656 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94282656 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7527772]\n +- Union\n :- InMemoryTableScan [date#94282656, numcos#94282679]\n : +- InMemoryRelation [date#94282656, overall#94282662, ret#94282666, resret#94282668, retnet#94282671, turnover#94282675, numcos#94282679, benchmark#94282682, excess_ret#94282685, excess_resret#94282708, excess_retnet#94282772], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94282340 = null) THEN null ELSE cast(date#94282340 as date) END AS date#94282656, CASE WHEN ((overall#94282341 = NA) OR (overall#94282341 = null)) THEN null ELSE cast(overall#94282341 as int) END AS overall#94282662, CASE WHEN ((ret#94282342 = NA) OR (ret#94282342 = null)) THEN null ELSE cast(ret#94282342 as float) END AS ret#94282666, CASE WHEN ((resret#94282343 = NA) OR (resret#94282343 = null)) THEN null ELSE cast(resret#94282343 as float) END A... [3620953] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620954 [labelType="html" label="MapPartitionsRDD [3620954] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620955 [labelType="html" label="MapPartitionsRDD [3620955] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620956 [labelType="html" label="MapPartitionsRDD [3620956] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296481 { label="CollectLimit"; 3620957 [labelType="html" label="MapPartitionsRDD [3620957] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620958 [labelType="html" label="MapPartitionsRDD [3620958] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3620950->3620951; 3620951->3620952; 3620952->3620953; 3620953->3620954; 3620954->3620955; 3620955->3620956; 3620956->3620957; 3620957->3620958; }

3620949,3620950

3620953

digraph G { subgraph clusterstage_512528 { label="Stage 512528"; subgraph cluster6296481 { label="CollectLimit"; 3620959 [labelType="html" label="ShuffledRowRDD [3620959] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620960 [labelType="html" label="MapPartitionsRDD [3620960] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296523 { label="mapPartitionsInternal"; 3620961 [labelType="html" label="MapPartitionsRDD [3620961] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6296524 { label="map"; 3620962 [labelType="html" label="MapPartitionsRDD [3620962] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3620959->3620960; 3620960->3620961; 3620961->3620962; }

3620958,3620959

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512528	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:12:06	22 ms	1/1		52.6 KiB
512527	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94282656 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94282656 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7527772] +- Union :- InMemoryTableScan [date#94282656, numcos#94282679] : +- InMemoryRelation [date#94282656, overall#94282662, ret#94282666, resret#94282668, retnet#94282671, turnover#94282675, numcos#94282679, benchmark#94282682, excess_ret#94282685, excess_resret#94282708, excess_retnet#94282772], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94282340 = null) THEN null ELSE cast(date#94282340 as date) END AS date#94282656, CASE WHEN ((overall#94282341 = NA) OR (overall#94282341 = null)) THEN null ELSE cast(overall#94282341 as int) END AS overall#94282662, CASE WHEN ((ret#94282342 = NA) OR (ret#94282342 = null)) THEN null ELSE cast(ret#94282342 as float) END AS ret#94282666, CASE WHEN ((resret#94282343 = NA) OR (resret#94282343 = null)) THEN null ELSE cast(resret#94282343 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:12:06	0.8 s	200/200		52.6 KiB	52.6 KiB
512526	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94282340 = null) THEN null ELSE cast(date#94282340 as date) END AS date#94282656, CASE WHEN ((overall#94282341 = NA) OR (overall#94282341 = null)) THEN null ELSE cast(overall#94282341 as int) END AS overall#94282662, CASE WHEN ((ret#94282342 = NA) OR (ret#94282342 = null)) THEN null ELSE cast(ret#94282342 as float) END AS ret#94282666, CASE WHEN ((resret#94282343 = NA) OR (resret#94282343 = null)) THEN null ELSE cast(resret#94282343 as float) END AS resret#94282668, CASE WHEN ((retnet#94282344 = NA) OR (retnet#94282344 = null)) THEN null ELSE cast(retnet#94282344 as float) END AS retnet#94282671, CASE WHEN ((turnover#94282345 = NA) OR (turnover#94282345 = null)) THEN null ELSE cast(turnover#94282345 as float) END AS turnover#94282675, CASE WHEN ((numcos#94282346 = NA) OR (numcos#94282346 = null)) THEN null ELSE cast(numcos#94282346 as float) END AS numcos#94282679, CASE WHEN ((benchmark#94282347 = NA) OR (benchmark#94282347 = null)) THEN null ELSE cast(benchmark#94282347 as fl...(1) Project [CASE WHEN (date#94282421 = null) THEN null ELSE cast(date#94282421 as date) END AS date#94282548, CASE WHEN ((cap#94282422 = NA) OR (cap#94282422 = null)) THEN null ELSE cast(cap#94282422 as float) END AS cap#94282549, CASE WHEN ((ret#94282423 = NA) OR (ret#94282423 = null)) THEN null ELSE cast(ret#94282423 as float) END AS ret#94282602, CASE WHEN ((resret#94282424 = NA) OR (resret#94282424 = null)) THEN null ELSE cast(resret#94282424 as float) END AS resret#94282603, CASE WHEN ((retnet#94282425 = NA) OR (retnet#94282425 = null)) THEN null ELSE cast(retnet#94282425 as float) END AS retnet#94282615, CASE WHEN ((turnover#94282426 = NA) OR (turnover#94282426 = null)) THEN null ELSE cast(turnover#94282426 as float) END AS turnover#94282653, CASE WHEN ((numcos#94282427 = NA) OR (numcos#94282427 = null)) THEN null ELSE cast(numcos#94282427 as float) END AS numcos#94282655, CASE WHEN ((coverage#94282428 = NA) OR (coverage#94282428 = null)) THEN null ELSE cast(coverage#94282428 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:12:06	41 ms	2/2	1000.8 KiB		52.6 KiB

Page: