PlusAMP App Middleware - Details for Job 330587

Status: SUCCEEDED
Submitted: 2025/08/01 18:52:38
Duration: 85 ms
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_511716 { label="Stage 511716"; subgraph cluster6278393 { label="Scan csv "; 3610548 [labelType="html" label="FileScanRDD [3610548] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610549 [labelType="html" label="MapPartitionsRDD [3610549] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278390 { label="WholeStageCodegen (1)"; 3610550 [labelType="html" label="MapPartitionsRDD [3610550] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278389 { label="InMemoryTableScan"; 3610551 [labelType="html" label="MapPartitionsRDD [3610551] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610552 [labelType="html" label="*(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS cap#94010646, CASE WHEN ((ret#94010361 = NA) OR (ret#94010361 = null)) THEN null ELSE cast(ret#94010361 as float) END AS ret#94010650, CASE WHEN ((resret#94010362 = NA) OR (resret#94010362 = null)) THEN null ELSE cast(resret#94010362 as float) END AS resret#94010656, CASE WHEN ((retnet#94010363 = NA) OR (retnet#94010363 = null)) THEN null ELSE cast(retnet#94010363 as float) END AS retnet#94010660, CASE WHEN ((turnover#94010364 = NA) OR (turnover#94010364 = null)) THEN null ELSE cast(turnover#94010364 as float) END AS turnover#94010664, CASE WHEN ((numcos#94010365 = NA) OR (numcos#94010365 = null)) THEN null ELSE cast(numcos#94010365 as float) END AS numcos#94010669, CASE WHEN ((coverage#94010366 = NA) OR (coverage#94010366 = null)) THEN null ELSE cast(coverage#94010366 as float) END AS cover... [3610552] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286485 { label="InMemoryTableScan"; 3615193 [labelType="html" label="MapPartitionsRDD [3615193] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615194 [labelType="html" label="MapPartitionsRDD [3615194] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615196 [labelType="html" label="MapPartitionsRDD [3615196] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286481 { label="WholeStageCodegen (1)"; 3615198 [labelType="html" label="MapPartitionsRDD [3615198] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286480 { label="Exchange"; 3615238 [labelType="html" label="MapPartitionsRDD [3615238] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3610548->3610549; 3610549->3610550; 3610550->3610551; 3610551->3610552; 3610552->3615193; 3615193->3615194; 3615194->3615196; 3615196->3615198; 3615198->3615238; }

3610552

digraph G { subgraph clusterstage_511717 { label="Stage 511717"; subgraph cluster6286480 { label="Exchange"; 3615239 [labelType="html" label="ShuffledRowRDD [3615239] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286477 { label="WholeStageCodegen (2)"; 3615240 [labelType="html" label="MapPartitionsRDD [3615240] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286476 { label="InMemoryTableScan"; 3615241 [labelType="html" label="MapPartitionsRDD [3615241] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615242 [labelType="html" label="*(2) Sort [date#94010641 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(date#94010641 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7515934]\n +- *(1) Project [date#94010643 AS date#94010641, numcos#94010669 AS numcos#94010663]\n +- *(1) Filter (isnotnull(cap#94010646) AND (cast(cap#94010646 as string) = 1))\n +- InMemoryTableScan [cap#94010646, date#94010643, numcos#94010669], [isnotnull(cap#94010646), (cast(cap#94010646 as string) = 1)]\n +- InMemoryRelation [date#94010643, cap#94010646, ret#94010650, resret#94010656, retnet#94010660, turnover#94010664, numcos#94010669, coverage#94010673, benchmark#94010677, excess_ret#94010681, excess_resret#94010683, excess_retnet#94010698], StorageLevel(disk, memory, deserialized, 1 replicas)\n +- *(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS ... [3615242] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615243 [labelType="html" label="MapPartitionsRDD [3615243] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615244 [labelType="html" label="MapPartitionsRDD [3615244] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615245 [labelType="html" label="MapPartitionsRDD [3615245] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286475 { label="CollectLimit"; 3615246 [labelType="html" label="MapPartitionsRDD [3615246] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286554 { label="mapPartitionsInternal"; 3615247 [labelType="html" label="MapPartitionsRDD [3615247] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286555 { label="map"; 3615248 [labelType="html" label="MapPartitionsRDD [3615248] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3615239->3615240; 3615240->3615241; 3615241->3615242; 3615242->3615243; 3615243->3615244; 3615244->3615245; 3615245->3615246; 3615246->3615247; 3615247->3615248; }

3615238,3615239

3615242

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
511717	default	toStream at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [date#94010641 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(date#94010641 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7515934] +- (1) Project [date#94010643 AS date#94010641, numcos#94010669 AS numcos#94010663] +- (1) Filter (isnotnull(cap#94010646) AND (cast(cap#94010646 as string) = 1)) +- InMemoryTableScan [cap#94010646, date#94010643, numcos#94010669], [isnotnull(cap#94010646), (cast(cap#94010646 as string) = 1)] +- InMemoryRelation [date#94010643, cap#94010646, ret#94010650, resret#94010656, retnet#94010660, turnover#94010664, numcos#94010669, coverage#94010673, benchmark#94010677, excess_ret#94010681, excess_resret#94010683, excess_retnet#94010698], StorageLevel(disk, memory, deserialized, 1 replicas) +- (1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS ... scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:52:38	32 ms	1/1
511716	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN (date#94010359 = null) THEN null ELSE cast(date#94010359 as date) END AS date#94010643, CASE WHEN ((cap#94010360 = NA) OR (cap#94010360 = null)) THEN null ELSE cast(cap#94010360 as float) END AS cap#94010646, CASE WHEN ((ret#94010361 = NA) OR (ret#94010361 = null)) THEN null ELSE cast(ret#94010361 as float) END AS ret#94010650, CASE WHEN ((resret#94010362 = NA) OR (resret#94010362 = null)) THEN null ELSE cast(resret#94010362 as float) END AS resret#94010656, CASE WHEN ((retnet#94010363 = NA) OR (retnet#94010363 = null)) THEN null ELSE cast(retnet#94010363 as float) END AS retnet#94010660, CASE WHEN ((turnover#94010364 = NA) OR (turnover#94010364 = null)) THEN null ELSE cast(turnover#94010364 as float) END AS turnover#94010664, CASE WHEN ((numcos#94010365 = NA) OR (numcos#94010365 = null)) THEN null ELSE cast(numcos#94010365 as float) END AS numcos#94010669, CASE WHEN ((coverage#94010366 = NA) OR (coverage#94010366 = null)) THEN null ELSE cast(coverage#94010366 as float) END AS cover... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:52:38	20 ms	1/1	262.7 KiB

Page: