PlusAMP App Middleware - Details for Job 330602

Status: SUCCEEDED
Submitted: 2025/08/01 18:52:41
Duration: 93 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_511748 { label="Stage 511748"; subgraph cluster6278393 { label="Scan csv "; 3610548 [labelType="html" label="FileScanRDD [3610548] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610549 [labelType="html" label="MapPartitionsRDD [3610549] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278390 { label="WholeStageCodegen (1)"; 3610550 [labelType="html" label="MapPartitionsRDD [3610550] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278389 { label="InMemoryTableScan"; 3610551 [labelType="html" label="MapPartitionsRDD [3610551] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610552 [labelType="html" label="*(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS cap#94010646, CASE WHEN ((ret#94010361 = NA) OR (ret#94010361 = null)) THEN null ELSE cast(ret#94010361 as float) END AS ret#94010650, CASE WHEN ((resret#94010362 = NA) OR (resret#94010362 = null)) THEN null ELSE cast(resret#94010362 as float) END AS resret#94010656, CASE WHEN ((retnet#94010363 = NA) OR (retnet#94010363 = null)) THEN null ELSE cast(retnet#94010363 as float) END AS retnet#94010660, CASE WHEN ((turnover#94010364 = NA) OR (turnover#94010364 = null)) THEN null ELSE cast(turnover#94010364 as float) END AS turnover#94010664, CASE WHEN ((numcos#94010365 = NA) OR (numcos#94010365 = null)) THEN null ELSE cast(numcos#94010365 as float) END AS numcos#94010669, CASE WHEN ((coverage#94010366 = NA) OR (coverage#94010366 = null)) THEN null ELSE cast(coverage#94010366 as float) END AS cover... [3610552] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286808 { label="InMemoryTableScan"; 3615368 [labelType="html" label="MapPartitionsRDD [3615368] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615369 [labelType="html" label="MapPartitionsRDD [3615369] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615370 [labelType="html" label="MapPartitionsRDD [3615370] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286804 { label="WholeStageCodegen (1)"; 3615371 [labelType="html" label="MapPartitionsRDD [3615371] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286803 { label="Exchange"; 3615406 [labelType="html" label="MapPartitionsRDD [3615406] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3610548->3610549; 3610549->3610550; 3610550->3610551; 3610551->3610552; 3610552->3615368; 3615368->3615369; 3615369->3615370; 3615370->3615371; 3615371->3615406; }

3610552

digraph G { subgraph clusterstage_511749 { label="Stage 511749"; subgraph cluster6286803 { label="Exchange"; 3615407 [labelType="html" label="ShuffledRowRDD [3615407] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286800 { label="WholeStageCodegen (2)"; 3615408 [labelType="html" label="MapPartitionsRDD [3615408] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286799 { label="InMemoryTableScan"; 3615409 [labelType="html" label="MapPartitionsRDD [3615409] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615411 [labelType="html" label="*(2) Sort [date#94010641 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94010641 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7516160]\n +- *(1) Project [date#94010643 AS date#94010641, numcos#94010669 AS numcos#94010663]\n +- *(1) Filter (isnotnull(cap#94010646) AND (cast(cap#94010646 as string) = 2))\n +- InMemoryTableScan [cap#94010646, date#94010643, numcos#94010669], [isnotnull(cap#94010646), (cast(cap#94010646 as string) = 2)]\n +- InMemoryRelation [date#94010643, cap#94010646, ret#94010650, resret#94010656, retnet#94010660, turnover#94010664, numcos#94010669, coverage#94010673, benchmark#94010677, excess_ret#94010681, excess_resret#94010683, excess_retnet#94010698], StorageLevel(disk, memory, deserialized, 1 replicas)\n +- *(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS ... [3615411] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615412 [labelType="html" label="MapPartitionsRDD [3615412] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615413 [labelType="html" label="MapPartitionsRDD [3615413] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615414 [labelType="html" label="MapPartitionsRDD [3615414] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286798 { label="CollectLimit"; 3615415 [labelType="html" label="MapPartitionsRDD [3615415] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286863 { label="mapPartitionsInternal"; 3615416 [labelType="html" label="MapPartitionsRDD [3615416] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286864 { label="map"; 3615417 [labelType="html" label="MapPartitionsRDD [3615417] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3615407->3615408; 3615408->3615409; 3615409->3615411; 3615411->3615412; 3615412->3615413; 3615413->3615414; 3615414->3615415; 3615415->3615416; 3615416->3615417; }

3615406,3615407

3615411

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
511749	default	toStream at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94010641 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94010641 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7516160] +- (1) Project [date#94010643 AS date#94010641, numcos#94010669 AS numcos#94010663] +- (1) Filter (isnotnull(cap#94010646) AND (cast(cap#94010646 as string) = 2)) +- InMemoryTableScan [cap#94010646, date#94010643, numcos#94010669], [isnotnull(cap#94010646), (cast(cap#94010646 as string) = 2)] +- InMemoryRelation [date#94010643, cap#94010646, ret#94010650, resret#94010656, retnet#94010660, turnover#94010664, numcos#94010669, coverage#94010673, benchmark#94010677, excess_ret#94010681, excess_resret#94010683, excess_retnet#94010698], StorageLevel(disk, memory, deserialized, 1 replicas) +- (1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS ... scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:52:41	26 ms	1/1
511748	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS cap#94010646, CASE WHEN ((ret#94010361 = NA) OR (ret#94010361 = null)) THEN null ELSE cast(ret#94010361 as float) END AS ret#94010650, CASE WHEN ((resret#94010362 = NA) OR (resret#94010362 = null)) THEN null ELSE cast(resret#94010362 as float) END AS resret#94010656, CASE WHEN ((retnet#94010363 = NA) OR (retnet#94010363 = null)) THEN null ELSE cast(retnet#94010363 as float) END AS retnet#94010660, CASE WHEN ((turnover#94010364 = NA) OR (turnover#94010364 = null)) THEN null ELSE cast(turnover#94010364 as float) END AS turnover#94010664, CASE WHEN ((numcos#94010365 = NA) OR (numcos#94010365 = null)) THEN null ELSE cast(numcos#94010365 as float) END AS numcos#94010669, CASE WHEN ((coverage#94010366 = NA) OR (coverage#94010366 = null)) THEN null ELSE cast(coverage#94010366 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:52:41	26 ms	1/1	262.7 KiB

Page: