PlusAMP App Middleware - Details for Job 331306

Status: SUCCEEDED
Submitted: 2025/08/02 15:18:59
Duration: 0.8 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512822 { label="Stage 512822"; subgraph cluster6300257 { label="Scan csv "; 3623094 [labelType="html" label="FileScanRDD [3623094] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623095 [labelType="html" label="MapPartitionsRDD [3623095] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300254 { label="WholeStageCodegen (1)"; 3623096 [labelType="html" label="MapPartitionsRDD [3623096] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300253 { label="InMemoryTableScan"; 3623097 [labelType="html" label="MapPartitionsRDD [3623097] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623098 [labelType="html" label="*(1) Project [CASE WHEN (date#94338953 = null) THEN null ELSE cast(date#94338953 as date) END AS date#94339134, CASE WHEN ((overall#94338954 = NA) OR (overall#94338954 = null)) THEN null ELSE cast(overall#94338954 as int) END AS overall#94339136, CASE WHEN ((ret#94338955 = NA) OR (ret#94338955 = null)) THEN null ELSE cast(ret#94338955 as float) END AS ret#94339138, CASE WHEN ((resret#94338956 = NA) OR (resret#94338956 = null)) THEN null ELSE cast(resret#94338956 as float) END AS resret#94339166, CASE WHEN ((retnet#94338957 = NA) OR (retnet#94338957 = null)) THEN null ELSE cast(retnet#94338957 as float) END AS retnet#94339168, CASE WHEN ((turnover#94338958 = NA) OR (turnover#94338958 = null)) THEN null ELSE cast(turnover#94338958 as float) END AS turnover#94339196, CASE WHEN ((numcos#94338959 = NA) OR (numcos#94338959 = null)) THEN null ELSE cast(numcos#94338959 as float) END AS numcos#94339198, CASE WHEN ((benchmark#94338960 = NA) OR (benchmark#94338960 = null)) THEN null ELSE cast(benchmark#94338960 as fl... [3623098] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623099 [labelType="html" label="MapPartitionsRDD [3623099] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623100 [labelType="html" label="MapPartitionsRDD [3623100] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623101 [labelType="html" label="MapPartitionsRDD [3623101] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300266 { label="Scan csv "; 3623102 [labelType="html" label="FileScanRDD [3623102] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623103 [labelType="html" label="MapPartitionsRDD [3623103] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300263 { label="WholeStageCodegen (1)"; 3623104 [labelType="html" label="MapPartitionsRDD [3623104] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300262 { label="InMemoryTableScan"; 3623105 [labelType="html" label="MapPartitionsRDD [3623105] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623106 [labelType="html" label="*(1) Project [CASE WHEN (date#94338925 = null) THEN null ELSE cast(date#94338925 as date) END AS date#94339106, CASE WHEN ((cap#94338926 = NA) OR (cap#94338926 = null)) THEN null ELSE cast(cap#94338926 as float) END AS cap#94339107, CASE WHEN ((ret#94338927 = NA) OR (ret#94338927 = null)) THEN null ELSE cast(ret#94338927 as float) END AS ret#94339108, CASE WHEN ((resret#94338928 = NA) OR (resret#94338928 = null)) THEN null ELSE cast(resret#94338928 as float) END AS resret#94339120, CASE WHEN ((retnet#94338929 = NA) OR (retnet#94338929 = null)) THEN null ELSE cast(retnet#94338929 as float) END AS retnet#94339121, CASE WHEN ((turnover#94338930 = NA) OR (turnover#94338930 = null)) THEN null ELSE cast(turnover#94338930 as float) END AS turnover#94339133, CASE WHEN ((numcos#94338931 = NA) OR (numcos#94338931 = null)) THEN null ELSE cast(numcos#94338931 as float) END AS numcos#94339135, CASE WHEN ((coverage#94338932 = NA) OR (coverage#94338932 = null)) THEN null ELSE cast(coverage#94338932 as float) END AS cover... [3623106] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623107 [labelType="html" label="MapPartitionsRDD [3623107] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623108 [labelType="html" label="MapPartitionsRDD [3623108] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623109 [labelType="html" label="MapPartitionsRDD [3623109] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300258 { label="WholeStageCodegen (1)"; 3623110 [labelType="html" label="MapPartitionsRDD [3623110] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300252 { label="Union"; 3623111 [labelType="html" label="UnionRDD [3623111] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300251 { label="Exchange"; 3623115 [labelType="html" label="MapPartitionsRDD [3623115] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623094->3623095; 3623095->3623096; 3623096->3623097; 3623097->3623098; 3623098->3623099; 3623099->3623100; 3623100->3623101; 3623102->3623103; 3623103->3623104; 3623104->3623105; 3623105->3623106; 3623106->3623107; 3623107->3623108; 3623108->3623109; 3623109->3623110; 3623101->3623111; 3623110->3623111; 3623111->3623115; }

3623098

3623106

digraph G { subgraph clusterstage_512823 { label="Stage 512823"; subgraph cluster6300251 { label="Exchange"; 3623116 [labelType="html" label="ShuffledRowRDD [3623116] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300248 { label="WholeStageCodegen (2)"; 3623117 [labelType="html" label="MapPartitionsRDD [3623117] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300247 { label="InMemoryTableScan"; 3623118 [labelType="html" label="MapPartitionsRDD [3623118] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623119 [labelType="html" label="*(2) Sort [date#94339134 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94339134 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532361]\n +- Union\n :- InMemoryTableScan [date#94339134, numcos#94339198]\n : +- InMemoryRelation [date#94339134, overall#94339136, ret#94339138, resret#94339166, retnet#94339168, turnover#94339196, numcos#94339198, benchmark#94339263, excess_ret#94339264, excess_resret#94339265, excess_retnet#94339266], StorageLevel(disk, memory, deserialized, 1 replicas)\n : +- *(1) Project [CASE WHEN (date#94338953 = null) THEN null ELSE cast(date#94338953 as date) END AS date#94339134, CASE WHEN ((overall#94338954 = NA) OR (overall#94338954 = null)) THEN null ELSE cast(overall#94338954 as int) END AS overall#94339136, CASE WHEN ((ret#94338955 = NA) OR (ret#94338955 = null)) THEN null ELSE cast(ret#94338955 as float) END AS ret#94339138, CASE WHEN ((resret#94338956 = NA) OR (resret#94338956 = null)) THEN null ELSE cast(resret#94338956 as float) END A... [3623119] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623120 [labelType="html" label="MapPartitionsRDD [3623120] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623121 [labelType="html" label="MapPartitionsRDD [3623121] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623122 [labelType="html" label="MapPartitionsRDD [3623122] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300246 { label="CollectLimit"; 3623123 [labelType="html" label="MapPartitionsRDD [3623123] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623124 [labelType="html" label="MapPartitionsRDD [3623124] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623116->3623117; 3623117->3623118; 3623118->3623119; 3623119->3623120; 3623120->3623121; 3623121->3623122; 3623122->3623123; 3623123->3623124; }

3623115,3623116

3623119

digraph G { subgraph clusterstage_512824 { label="Stage 512824"; subgraph cluster6300246 { label="CollectLimit"; 3623125 [labelType="html" label="ShuffledRowRDD [3623125] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623126 [labelType="html" label="MapPartitionsRDD [3623126] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300288 { label="mapPartitionsInternal"; 3623127 [labelType="html" label="MapPartitionsRDD [3623127] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300289 { label="map"; 3623128 [labelType="html" label="MapPartitionsRDD [3623128] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623125->3623126; 3623126->3623127; 3623127->3623128; }

3623124,3623125

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512824	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:19:00	19 ms	1/1		34.2 KiB
512823	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94339134 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94339134 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532361] +- Union :- InMemoryTableScan [date#94339134, numcos#94339198] : +- InMemoryRelation [date#94339134, overall#94339136, ret#94339138, resret#94339166, retnet#94339168, turnover#94339196, numcos#94339198, benchmark#94339263, excess_ret#94339264, excess_resret#94339265, excess_retnet#94339266], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN (date#94338953 = null) THEN null ELSE cast(date#94338953 as date) END AS date#94339134, CASE WHEN ((overall#94338954 = NA) OR (overall#94338954 = null)) THEN null ELSE cast(overall#94338954 as int) END AS overall#94339136, CASE WHEN ((ret#94338955 = NA) OR (ret#94338955 = null)) THEN null ELSE cast(ret#94338955 as float) END AS ret#94339138, CASE WHEN ((resret#94338956 = NA) OR (resret#94338956 = null)) THEN null ELSE cast(resret#94338956 as float) END A... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:18:59	0.8 s	200/200		34.2 KiB	34.2 KiB
512822	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN (date#94338953 = null) THEN null ELSE cast(date#94338953 as date) END AS date#94339134, CASE WHEN ((overall#94338954 = NA) OR (overall#94338954 = null)) THEN null ELSE cast(overall#94338954 as int) END AS overall#94339136, CASE WHEN ((ret#94338955 = NA) OR (ret#94338955 = null)) THEN null ELSE cast(ret#94338955 as float) END AS ret#94339138, CASE WHEN ((resret#94338956 = NA) OR (resret#94338956 = null)) THEN null ELSE cast(resret#94338956 as float) END AS resret#94339166, CASE WHEN ((retnet#94338957 = NA) OR (retnet#94338957 = null)) THEN null ELSE cast(retnet#94338957 as float) END AS retnet#94339168, CASE WHEN ((turnover#94338958 = NA) OR (turnover#94338958 = null)) THEN null ELSE cast(turnover#94338958 as float) END AS turnover#94339196, CASE WHEN ((numcos#94338959 = NA) OR (numcos#94338959 = null)) THEN null ELSE cast(numcos#94338959 as float) END AS numcos#94339198, CASE WHEN ((benchmark#94338960 = NA) OR (benchmark#94338960 = null)) THEN null ELSE cast(benchmark#94338960 as fl...(1) Project [CASE WHEN (date#94338925 = null) THEN null ELSE cast(date#94338925 as date) END AS date#94339106, CASE WHEN ((cap#94338926 = NA) OR (cap#94338926 = null)) THEN null ELSE cast(cap#94338926 as float) END AS cap#94339107, CASE WHEN ((ret#94338927 = NA) OR (ret#94338927 = null)) THEN null ELSE cast(ret#94338927 as float) END AS ret#94339108, CASE WHEN ((resret#94338928 = NA) OR (resret#94338928 = null)) THEN null ELSE cast(resret#94338928 as float) END AS resret#94339120, CASE WHEN ((retnet#94338929 = NA) OR (retnet#94338929 = null)) THEN null ELSE cast(retnet#94338929 as float) END AS retnet#94339121, CASE WHEN ((turnover#94338930 = NA) OR (turnover#94338930 = null)) THEN null ELSE cast(turnover#94338930 as float) END AS turnover#94339133, CASE WHEN ((numcos#94338931 = NA) OR (numcos#94338931 = null)) THEN null ELSE cast(numcos#94338931 as float) END AS numcos#94339135, CASE WHEN ((coverage#94338932 = NA) OR (coverage#94338932 = null)) THEN null ELSE cast(coverage#94338932 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/02 15:18:59	38 ms	2/2	577.9 KiB		34.2 KiB

Page: