PlusAMP App Middleware - Details for Job 331061

Status: SUCCEEDED
Submitted: 2025/08/02 15:10:23
Duration: 0.8 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512452 { label="Stage 512452"; subgraph cluster6295559 { label="Scan csv "; 3620391 [labelType="html" label="FileScanRDD [3620391] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620392 [labelType="html" label="MapPartitionsRDD [3620392] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295556 { label="WholeStageCodegen (1)"; 3620393 [labelType="html" label="MapPartitionsRDD [3620393] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295555 { label="InMemoryTableScan"; 3620394 [labelType="html" label="MapPartitionsRDD [3620394] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620395 [labelType="html" label="*(1) Project [CASE WHEN (date#94267773 = null) THEN null ELSE cast(date#94267773 as date) END AS date#94268081, CASE WHEN ((overall#94267774 = NA) OR (overall#94267774 = null)) THEN null ELSE cast(overall#94267774 as int) END AS overall#94268086, CASE WHEN ((ret#94267775 = NA) OR (ret#94267775 = null)) THEN null ELSE cast(ret#94267775 as float) END AS ret#94268089, CASE WHEN ((resret#94267776 = NA) OR (resret#94267776 = null)) THEN null ELSE cast(resret#94267776 as float) END AS resret#94268092, CASE WHEN ((retnet#94267777 = NA) OR (retnet#94267777 = null)) THEN null ELSE cast(retnet#94267777 as float) END AS retnet#94268098, CASE WHEN ((turnover#94267778 = NA) OR (turnover#94267778 = null)) THEN null ELSE cast(turnover#94267778 as float) END AS turnover#94268102, CASE WHEN ((numcos#94267779 = NA) OR (numcos#94267779 = null)) THEN null ELSE cast(numcos#94267779 as float) END AS numcos#94268104, CASE WHEN ((benchmark#94267780 = NA) OR (benchmark#94267780 = null)) THEN null ELSE cast(benchmark#94267780 as fl... [3620395] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620396 [labelType="html" label="MapPartitionsRDD [3620396] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620397 [labelType="html" label="MapPartitionsRDD [3620397] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620398 [labelType="html" label="MapPartitionsRDD [3620398] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295568 { label="Scan csv "; 3620399 [labelType="html" label="FileScanRDD [3620399] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620400 [labelType="html" label="MapPartitionsRDD [3620400] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295565 { label="WholeStageCodegen (1)"; 3620401 [labelType="html" label="MapPartitionsRDD [3620401] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295564 { label="InMemoryTableScan"; 3620402 [labelType="html" label="MapPartitionsRDD [3620402] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620403 [labelType="html" label="*(1) Project [CASE WHEN (date#94267801 = null) THEN null ELSE cast(date#94267801 as date) END AS date#94268082, CASE WHEN ((cap#94267802 = NA) OR (cap#94267802 = null)) THEN null ELSE cast(cap#94267802 as float) END AS cap#94268084, CASE WHEN ((ret#94267803 = NA) OR (ret#94267803 = null)) THEN null ELSE cast(ret#94267803 as float) END AS ret#94268088, CASE WHEN ((resret#94267804 = NA) OR (resret#94267804 = null)) THEN null ELSE cast(resret#94267804 as float) END AS resret#94268093, CASE WHEN ((retnet#94267805 = NA) OR (retnet#94267805 = null)) THEN null ELSE cast(retnet#94267805 as float) END AS retnet#94268097, CASE WHEN ((turnover#94267806 = NA) OR (turnover#94267806 = null)) THEN null ELSE cast(turnover#94267806 as float) END AS turnover#94268101, CASE WHEN ((numcos#94267807 = NA) OR (numcos#94267807 = null)) THEN null ELSE cast(numcos#94267807 as float) END AS numcos#94268106, CASE WHEN ((coverage#94267808 = NA) OR (coverage#94267808 = null)) THEN null ELSE cast(coverage#94267808 as float) END AS cover... [3620403] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620404 [labelType="html" label="MapPartitionsRDD [3620404] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620405 [labelType="html" label="MapPartitionsRDD [3620405] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620406 [labelType="html" label="MapPartitionsRDD [3620406] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295560 { label="WholeStageCodegen (1)"; 3620407 [labelType="html" label="MapPartitionsRDD [3620407] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295554 { label="Union"; 3620408 [labelType="html" label="UnionRDD [3620408] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295553 { label="Exchange"; 3620412 [labelType="html" label="MapPartitionsRDD [3620412] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3620391->3620392; 3620392->3620393; 3620393->3620394; 3620394->3620395; 3620395->3620396; 3620396->3620397; 3620397->3620398; 3620399->3620400; 3620400->3620401; 3620401->3620402; 3620402->3620403; 3620403->3620404; 3620404->3620405; 3620405->3620406; 3620406->3620407; 3620398->3620408; 3620407->3620408; 3620408->3620412; }

3620395

3620403

digraph G { subgraph clusterstage_512453 { label="Stage 512453"; subgraph cluster6295553 { label="Exchange"; 3620413 [labelType="html" label="ShuffledRowRDD [3620413] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295550 { label="WholeStageCodegen (2)"; 3620414 [labelType="html" label="MapPartitionsRDD [3620414] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295549 { label="InMemoryTableScan"; 3620415 [labelType="html" label="MapPartitionsRDD [3620415] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620416 [labelType="html" label="*(2) Sort [date#94268081 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94268081 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7526624]\n +- Union\n :- InMemoryTableScan [date#94268081, numcos#94268104]\n : +- InMemoryRelation [date#94268081, overall#94268086, ret#94268089, resret#94268092, retnet#94268098, turnover#94268102, numcos#94268104, benchmark#94268108, excess_ret#94268113, excess_resret#94268117, excess_retnet#94268121], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94267773 = null) THEN null ELSE cast(date#94267773 as date) END AS date#94268081, CASE WHEN ((overall#94267774 = NA) OR (overall#94267774 = null)) THEN null ELSE cast(overall#94267774 as int) END AS overall#94268086, CASE WHEN ((ret#94267775 = NA) OR (ret#94267775 = null)) THEN null ELSE cast(ret#94267775 as float) END AS ret#94268089, CASE WHEN ((resret#94267776 = NA) OR (resret#94267776 = null)) THEN null ELSE cast(resret#94267776 as float) END A... [3620416] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620417 [labelType="html" label="MapPartitionsRDD [3620417] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620418 [labelType="html" label="MapPartitionsRDD [3620418] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620419 [labelType="html" label="MapPartitionsRDD [3620419] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295548 { label="CollectLimit"; 3620420 [labelType="html" label="MapPartitionsRDD [3620420] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620421 [labelType="html" label="MapPartitionsRDD [3620421] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3620413->3620414; 3620414->3620415; 3620415->3620416; 3620416->3620417; 3620417->3620418; 3620418->3620419; 3620419->3620420; 3620420->3620421; }

3620412,3620413

3620416

digraph G { subgraph clusterstage_512454 { label="Stage 512454"; subgraph cluster6295548 { label="CollectLimit"; 3620422 [labelType="html" label="ShuffledRowRDD [3620422] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3620423 [labelType="html" label="MapPartitionsRDD [3620423] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295590 { label="mapPartitionsInternal"; 3620424 [labelType="html" label="MapPartitionsRDD [3620424] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6295591 { label="map"; 3620425 [labelType="html" label="MapPartitionsRDD [3620425] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3620422->3620423; 3620423->3620424; 3620424->3620425; }

3620421,3620422

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512454	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:10:24	17 ms	1/1		36.5 KiB
512453	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94268081 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94268081 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7526624] +- Union :- InMemoryTableScan [date#94268081, numcos#94268104] : +- InMemoryRelation [date#94268081, overall#94268086, ret#94268089, resret#94268092, retnet#94268098, turnover#94268102, numcos#94268104, benchmark#94268108, excess_ret#94268113, excess_resret#94268117, excess_retnet#94268121], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94267773 = null) THEN null ELSE cast(date#94267773 as date) END AS date#94268081, CASE WHEN ((overall#94267774 = NA) OR (overall#94267774 = null)) THEN null ELSE cast(overall#94267774 as int) END AS overall#94268086, CASE WHEN ((ret#94267775 = NA) OR (ret#94267775 = null)) THEN null ELSE cast(ret#94267775 as float) END AS ret#94268089, CASE WHEN ((resret#94267776 = NA) OR (resret#94267776 = null)) THEN null ELSE cast(resret#94267776 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:10:23	0.7 s	200/200		36.5 KiB	36.5 KiB
512452	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94267773 = null) THEN null ELSE cast(date#94267773 as date) END AS date#94268081, CASE WHEN ((overall#94267774 = NA) OR (overall#94267774 = null)) THEN null ELSE cast(overall#94267774 as int) END AS overall#94268086, CASE WHEN ((ret#94267775 = NA) OR (ret#94267775 = null)) THEN null ELSE cast(ret#94267775 as float) END AS ret#94268089, CASE WHEN ((resret#94267776 = NA) OR (resret#94267776 = null)) THEN null ELSE cast(resret#94267776 as float) END AS resret#94268092, CASE WHEN ((retnet#94267777 = NA) OR (retnet#94267777 = null)) THEN null ELSE cast(retnet#94267777 as float) END AS retnet#94268098, CASE WHEN ((turnover#94267778 = NA) OR (turnover#94267778 = null)) THEN null ELSE cast(turnover#94267778 as float) END AS turnover#94268102, CASE WHEN ((numcos#94267779 = NA) OR (numcos#94267779 = null)) THEN null ELSE cast(numcos#94267779 as float) END AS numcos#94268104, CASE WHEN ((benchmark#94267780 = NA) OR (benchmark#94267780 = null)) THEN null ELSE cast(benchmark#94267780 as fl...(1) Project [CASE WHEN (date#94267801 = null) THEN null ELSE cast(date#94267801 as date) END AS date#94268082, CASE WHEN ((cap#94267802 = NA) OR (cap#94267802 = null)) THEN null ELSE cast(cap#94267802 as float) END AS cap#94268084, CASE WHEN ((ret#94267803 = NA) OR (ret#94267803 = null)) THEN null ELSE cast(ret#94267803 as float) END AS ret#94268088, CASE WHEN ((resret#94267804 = NA) OR (resret#94267804 = null)) THEN null ELSE cast(resret#94267804 as float) END AS resret#94268093, CASE WHEN ((retnet#94267805 = NA) OR (retnet#94267805 = null)) THEN null ELSE cast(retnet#94267805 as float) END AS retnet#94268097, CASE WHEN ((turnover#94267806 = NA) OR (turnover#94267806 = null)) THEN null ELSE cast(turnover#94267806 as float) END AS turnover#94268101, CASE WHEN ((numcos#94267807 = NA) OR (numcos#94267807 = null)) THEN null ELSE cast(numcos#94267807 as float) END AS numcos#94268106, CASE WHEN ((coverage#94267808 = NA) OR (coverage#94267808 = null)) THEN null ELSE cast(coverage#94267808 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:10:23	58 ms	2/2	542.4 KiB		36.5 KiB

Page: