PlusAMP App Middleware - Details for Job 331512

Status: SUCCEEDED
Submitted: 2025/08/02 15:26:00
Duration: 0.1 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_513132 { label="Stage 513132"; subgraph cluster6304130 { label="Scan csv "; 3625325 [labelType="html" label="FileScanRDD [3625325] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625326 [labelType="html" label="MapPartitionsRDD [3625326] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304097 { label="WholeStageCodegen (1)"; 3625327 [labelType="html" label="MapPartitionsRDD [3625327] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304096 { label="InMemoryTableScan"; 3625328 [labelType="html" label="MapPartitionsRDD [3625328] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625329 [labelType="html" label="*(1) Project [CASE WHEN ((year#94399178 = NA) OR (year#94399178 = null)) THEN null ELSE cast(year#94399178 as int) END AS year#94399378, CASE WHEN ((retIC#94399179 = NA) OR (retIC#94399179 = null)) THEN null ELSE cast(retIC#94399179 as float) END AS retIC#94399379, CASE WHEN ((resretIC#94399180 = NA) OR (resretIC#94399180 = null)) THEN null ELSE cast(resretIC#94399180 as float) END AS resretIC#94399413, CASE WHEN ((numcos#94399181 = NA) OR (numcos#94399181 = null)) THEN null ELSE cast(numcos#94399181 as float) END AS numcos#94399414, CASE WHEN ((numdates#94399182 = NA) OR (numdates#94399182 = null)) THEN null ELSE cast(numdates#94399182 as int) END AS numdates#94399415, CASE WHEN ((annual_bmret#94399183 = NA) OR (annual_bmret#94399183 = null)) THEN null ELSE cast(annual_bmret#94399183 as float) END AS annual_bmret#94399416, CASE WHEN ((annual_ret#94399184 = NA) OR (annual_ret#94399184 = null)) THEN null ELSE cast(annual_ret#94399184 as float) END AS annual_ret#94399418, CASE WHEN ((std_ret#94399185 = NA) O... [3625329] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625330 [labelType="html" label="MapPartitionsRDD [3625330] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625331 [labelType="html" label="MapPartitionsRDD [3625331] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625332 [labelType="html" label="MapPartitionsRDD [3625332] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304091 { label="WholeStageCodegen (1)"; 3625333 [labelType="html" label="MapPartitionsRDD [3625333] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304090 { label="Exchange"; 3625371 [labelType="html" label="MapPartitionsRDD [3625371] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625325->3625326; 3625326->3625327; 3625327->3625328; 3625328->3625329; 3625329->3625330; 3625330->3625331; 3625331->3625332; 3625332->3625333; 3625333->3625371; }

3625329

digraph G { subgraph clusterstage_513133 { label="Stage 513133"; subgraph cluster6304090 { label="Exchange"; 3625372 [labelType="html" label="ShuffledRowRDD [3625372] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304087 { label="WholeStageCodegen (2)"; 3625373 [labelType="html" label="MapPartitionsRDD [3625373] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304086 { label="InMemoryTableScan"; 3625374 [labelType="html" label="MapPartitionsRDD [3625374] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625375 [labelType="html" label="*(2) Sort [year#94399378 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(year#94399378 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7537025]\n +- *(1) Project [year#94399378, turnover#94399453, (1.0 / cast(turnover#94399453 as double)) AS days_hold#94399494]\n +- *(1) Filter isnotnull(turnover#94399453)\n +- *(1) ColumnarToRow\n +- InMemoryTableScan [turnover#94399453, year#94399378], [isnotnull(turnover#94399453)]\n +- InMemoryRelation [year#94399378, retIC#94399379, resretIC#94399413, numcos#94399414, numdates#94399415, annual_bmret#94399416, annual_ret#94399418, std_ret#94399420, Sharpe_ret#94399422, PctPos_ret#94399423, TR_ret#94399426, IR_ret#94399427, annual_resret#94399430, std_resret#94399432, Sharpe_resret#94399434, PctPos_resret#94399436, TR_resret#94399437, IR_resret#94399439, annual_retnet#94399442, std_retnet#94399443, Sharpe_retnet#94399446, PctPos_retnet#94399447, TR_retnet#94399450, IR_retnet#94399451, turnover#94399453], StorageLevel(disk, ... [3625375] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625376 [labelType="html" label="MapPartitionsRDD [3625376] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625377 [labelType="html" label="MapPartitionsRDD [3625377] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625378 [labelType="html" label="MapPartitionsRDD [3625378] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304083 { label="WholeStageCodegen (1)"; 3625379 [labelType="html" label="MapPartitionsRDD [3625379] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304082 { label="CollectLimit"; 3625380 [labelType="html" label="MapPartitionsRDD [3625380] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625381 [labelType="html" label="MapPartitionsRDD [3625381] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625372->3625373; 3625373->3625374; 3625374->3625375; 3625375->3625376; 3625376->3625377; 3625377->3625378; 3625378->3625379; 3625379->3625380; 3625380->3625381; }

3625371,3625372

3625375

digraph G { subgraph clusterstage_513134 { label="Stage 513134"; subgraph cluster6304082 { label="CollectLimit"; 3625382 [labelType="html" label="ShuffledRowRDD [3625382] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3625383 [labelType="html" label="MapPartitionsRDD [3625383] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304206 { label="mapPartitionsInternal"; 3625384 [labelType="html" label="MapPartitionsRDD [3625384] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6304207 { label="map"; 3625385 [labelType="html" label="MapPartitionsRDD [3625385] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3625382->3625383; 3625383->3625384; 3625384->3625385; }

3625381,3625382

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
513134	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:26:01	10 ms	1/1		1846.0 B
513133	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [year#94399378 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94399378 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7537025] +- (1) Project [year#94399378, turnover#94399453, (1.0 / cast(turnover#94399453 as double)) AS days_hold#94399494] +- (1) Filter isnotnull(turnover#94399453) +- (1) ColumnarToRow +- InMemoryTableScan [turnover#94399453, year#94399378], [isnotnull(turnover#94399453)] +- InMemoryRelation [year#94399378, retIC#94399379, resretIC#94399413, numcos#94399414, numdates#94399415, annual_bmret#94399416, annual_ret#94399418, std_ret#94399420, Sharpe_ret#94399422, PctPos_ret#94399423, TR_ret#94399426, IR_ret#94399427, annual_resret#94399430, std_resret#94399432, Sharpe_resret#94399434, PctPos_resret#94399436, TR_resret#94399437, IR_resret#94399439, annual_retnet#94399442, std_retnet#94399443, Sharpe_retnet#94399446, PctPos_retnet#94399447, TR_retnet#94399450, IR_retnet#94399451, turnover#94399453], StorageLevel(disk, ... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:26:00	0.1 s	26/26		1846.0 B	1846.0 B
513132	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((year#94399178 = NA) OR (year#94399178 = null)) THEN null ELSE cast(year#94399178 as int) END AS year#94399378, CASE WHEN ((retIC#94399179 = NA) OR (retIC#94399179 = null)) THEN null ELSE cast(retIC#94399179 as float) END AS retIC#94399379, CASE WHEN ((resretIC#94399180 = NA) OR (resretIC#94399180 = null)) THEN null ELSE cast(resretIC#94399180 as float) END AS resretIC#94399413, CASE WHEN ((numcos#94399181 = NA) OR (numcos#94399181 = null)) THEN null ELSE cast(numcos#94399181 as float) END AS numcos#94399414, CASE WHEN ((numdates#94399182 = NA) OR (numdates#94399182 = null)) THEN null ELSE cast(numdates#94399182 as int) END AS numdates#94399415, CASE WHEN ((annual_bmret#94399183 = NA) OR (annual_bmret#94399183 = null)) THEN null ELSE cast(annual_bmret#94399183 as float) END AS annual_bmret#94399416, CASE WHEN ((annual_ret#94399184 = NA) OR (annual_ret#94399184 = null)) THEN null ELSE cast(annual_ret#94399184 as float) END AS annual_ret#94399418, CASE WHEN ((std_ret#94399185 = NA) O... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:26:00	13 ms	1/1	4.5 KiB		1846.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

513134

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:26:01

10 ms

1/1

1846.0 B

513133

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(2) Sort [year#94399378 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94399378 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7537025] +- *(1) Project [year#94399378, turnover#94399453, (1.0 / cast(turnover#94399453 as double)) AS days_hold#94399494] +- *(1) Filter isnotnull(turnover#94399453) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94399453, year#94399378], [isnotnull(turnover#94399453)] +- InMemoryRelation [year#94399378, retIC#94399379, resretIC#94399413, numcos#94399414, numdates#94399415, annual_bmret#94399416, annual_ret#94399418, std_ret#94399420, Sharpe_ret#94399422, PctPos_ret#94399423, TR_ret#94399426, IR_ret#94399427, annual_resret#94399430, std_resret#94399432, Sharpe_resret#94399434, PctPos_resret#94399436, TR_resret#94399437, IR_resret#94399439, annual_retnet#94399442, std_retnet#94399443, Sharpe_retnet#94399446, PctPos_retnet#94399447, TR_retnet#94399450, IR_retnet#94399451, turnover#94399453], StorageLevel(disk, ...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:26:00

0.1 s

26/26

1846.0 B

513132

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((year#94399178 = NA) OR (year#94399178 = null)) THEN null ELSE cast(year#94399178 as int) END AS year#94399378, CASE WHEN ((retIC#94399179 = NA) OR (retIC#94399179 = null)) THEN null ELSE cast(retIC#94399179 as float) END AS retIC#94399379, CASE WHEN ((resretIC#94399180 = NA) OR (resretIC#94399180 = null)) THEN null ELSE cast(resretIC#94399180 as float) END AS resretIC#94399413, CASE WHEN ((numcos#94399181 = NA) OR (numcos#94399181 = null)) THEN null ELSE cast(numcos#94399181 as float) END AS numcos#94399414, CASE WHEN ((numdates#94399182 = NA) OR (numdates#94399182 = null)) THEN null ELSE cast(numdates#94399182 as int) END AS numdates#94399415, CASE WHEN ((annual_bmret#94399183 = NA) OR (annual_bmret#94399183 = null)) THEN null ELSE cast(annual_bmret#94399183 as float) END AS annual_bmret#94399416, CASE WHEN ((annual_ret#94399184 = NA) OR (annual_ret#94399184 = null)) THEN null ELSE cast(annual_ret#94399184 as float) END AS annual_ret#94399418, CASE WHEN ((std_ret#94399185 = NA) O...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:26:00

13 ms

1/1

4.5 KiB

1846.0 B

Page: