PlusAMP App Middleware - Details for Job 330625

Status: SUCCEEDED
Submitted: 2025/08/01 18:52:46
Duration: 67 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_511792 { label="Stage 511792"; subgraph cluster6278393 { label="Scan csv "; 3610548 [labelType="html" label="FileScanRDD [3610548] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610549 [labelType="html" label="MapPartitionsRDD [3610549] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278390 { label="WholeStageCodegen (1)"; 3610550 [labelType="html" label="MapPartitionsRDD [3610550] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278389 { label="InMemoryTableScan"; 3610551 [labelType="html" label="MapPartitionsRDD [3610551] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610552 [labelType="html" label="*(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS cap#94010646, CASE WHEN ((ret#94010361 = NA) OR (ret#94010361 = null)) THEN null ELSE cast(ret#94010361 as float) END AS ret#94010650, CASE WHEN ((resret#94010362 = NA) OR (resret#94010362 = null)) THEN null ELSE cast(resret#94010362 as float) END AS resret#94010656, CASE WHEN ((retnet#94010363 = NA) OR (retnet#94010363 = null)) THEN null ELSE cast(retnet#94010363 as float) END AS retnet#94010660, CASE WHEN ((turnover#94010364 = NA) OR (turnover#94010364 = null)) THEN null ELSE cast(turnover#94010364 as float) END AS turnover#94010664, CASE WHEN ((numcos#94010365 = NA) OR (numcos#94010365 = null)) THEN null ELSE cast(numcos#94010365 as float) END AS numcos#94010669, CASE WHEN ((coverage#94010366 = NA) OR (coverage#94010366 = null)) THEN null ELSE cast(coverage#94010366 as float) END AS cover... [3610552] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287152 { label="InMemoryTableScan"; 3615558 [labelType="html" label="MapPartitionsRDD [3615558] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615559 [labelType="html" label="MapPartitionsRDD [3615559] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615560 [labelType="html" label="MapPartitionsRDD [3615560] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287146 { label="WholeStageCodegen (1)"; 3615561 [labelType="html" label="MapPartitionsRDD [3615561] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287145 { label="Exchange"; 3615594 [labelType="html" label="MapPartitionsRDD [3615594] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3610548->3610549; 3610549->3610550; 3610550->3610551; 3610551->3610552; 3610552->3615558; 3615558->3615559; 3615559->3615560; 3615560->3615561; 3615561->3615594; }

3610552

digraph G { subgraph clusterstage_511793 { label="Stage 511793"; subgraph cluster6287145 { label="Exchange"; 3615595 [labelType="html" label="ShuffledRowRDD [3615595] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287135 { label="WholeStageCodegen (2)"; 3615596 [labelType="html" label="MapPartitionsRDD [3615596] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287134 { label="InMemoryTableScan"; 3615597 [labelType="html" label="MapPartitionsRDD [3615597] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615598 [labelType="html" label="*(2) Sort [date#94010641 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94010641 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7516445]\n +- *(1) Project [date#94010643 AS date#94010641, numcos#94010669 AS numcos#94010663]\n +- *(1) Filter (isnotnull(cap#94010646) AND (cast(cap#94010646 as string) = 3))\n +- InMemoryTableScan [cap#94010646, date#94010643, numcos#94010669], [isnotnull(cap#94010646), (cast(cap#94010646 as string) = 3)]\n +- InMemoryRelation [date#94010643, cap#94010646, ret#94010650, resret#94010656, retnet#94010660, turnover#94010664, numcos#94010669, coverage#94010673, benchmark#94010677, excess_ret#94010681, excess_resret#94010683, excess_retnet#94010698], StorageLevel(disk, memory, deserialized, 1 replicas)\n +- *(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS ... [3615598] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615599 [labelType="html" label="MapPartitionsRDD [3615599] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615600 [labelType="html" label="MapPartitionsRDD [3615600] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615601 [labelType="html" label="MapPartitionsRDD [3615601] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287133 { label="CollectLimit"; 3615602 [labelType="html" label="MapPartitionsRDD [3615602] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287217 { label="mapPartitionsInternal"; 3615603 [labelType="html" label="MapPartitionsRDD [3615603] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6287218 { label="map"; 3615604 [labelType="html" label="MapPartitionsRDD [3615604] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3615595->3615596; 3615596->3615597; 3615597->3615598; 3615598->3615599; 3615599->3615600; 3615600->3615601; 3615601->3615602; 3615602->3615603; 3615603->3615604; }

3615594,3615595

3615598

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
511793	default	toStream at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94010641 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94010641 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7516445] +- (1) Project [date#94010643 AS date#94010641, numcos#94010669 AS numcos#94010663] +- (1) Filter (isnotnull(cap#94010646) AND (cast(cap#94010646 as string) = 3)) +- InMemoryTableScan [cap#94010646, date#94010643, numcos#94010669], [isnotnull(cap#94010646), (cast(cap#94010646 as string) = 3)] +- InMemoryRelation [date#94010643, cap#94010646, ret#94010650, resret#94010656, retnet#94010660, turnover#94010664, numcos#94010669, coverage#94010673, benchmark#94010677, excess_ret#94010681, excess_resret#94010683, excess_retnet#94010698], StorageLevel(disk, memory, deserialized, 1 replicas) +- (1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS ... scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:52:46	22 ms	1/1
511792	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS cap#94010646, CASE WHEN ((ret#94010361 = NA) OR (ret#94010361 = null)) THEN null ELSE cast(ret#94010361 as float) END AS ret#94010650, CASE WHEN ((resret#94010362 = NA) OR (resret#94010362 = null)) THEN null ELSE cast(resret#94010362 as float) END AS resret#94010656, CASE WHEN ((retnet#94010363 = NA) OR (retnet#94010363 = null)) THEN null ELSE cast(retnet#94010363 as float) END AS retnet#94010660, CASE WHEN ((turnover#94010364 = NA) OR (turnover#94010364 = null)) THEN null ELSE cast(turnover#94010364 as float) END AS turnover#94010664, CASE WHEN ((numcos#94010365 = NA) OR (numcos#94010365 = null)) THEN null ELSE cast(numcos#94010365 as float) END AS numcos#94010669, CASE WHEN ((coverage#94010366 = NA) OR (coverage#94010366 = null)) THEN null ELSE cast(coverage#94010366 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:52:46	16 ms	1/1	262.7 KiB

Page: