PlusAMP App Middleware - Details for Job 331414

Status: SUCCEEDED
Submitted: 2025/08/02 15:22:32
Duration: 0.2 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512984 { label="Stage 512984"; subgraph cluster6302271 { label="Scan csv "; 3624251 [labelType="html" label="FileScanRDD [3624251] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624252 [labelType="html" label="MapPartitionsRDD [3624252] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302228 { label="WholeStageCodegen (1)"; 3624253 [labelType="html" label="MapPartitionsRDD [3624253] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302227 { label="InMemoryTableScan"; 3624254 [labelType="html" label="MapPartitionsRDD [3624254] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624255 [labelType="html" label="*(1) Project [CASE WHEN ((year#94371422 = NA) OR (year#94371422 = null)) THEN null ELSE cast(year#94371422 as int) END AS year#94371588, CASE WHEN ((retIC#94371423 = NA) OR (retIC#94371423 = null)) THEN null ELSE cast(retIC#94371423 as float) END AS retIC#94371589, CASE WHEN ((resretIC#94371424 = NA) OR (resretIC#94371424 = null)) THEN null ELSE cast(resretIC#94371424 as float) END AS resretIC#94371590, CASE WHEN ((numcos#94371425 = NA) OR (numcos#94371425 = null)) THEN null ELSE cast(numcos#94371425 as float) END AS numcos#94371624, CASE WHEN ((numdates#94371426 = NA) OR (numdates#94371426 = null)) THEN null ELSE cast(numdates#94371426 as int) END AS numdates#94371625, CASE WHEN ((annual_bmret#94371427 = NA) OR (annual_bmret#94371427 = null)) THEN null ELSE cast(annual_bmret#94371427 as float) END AS annual_bmret#94371626, CASE WHEN ((annual_ret#94371428 = NA) OR (annual_ret#94371428 = null)) THEN null ELSE cast(annual_ret#94371428 as float) END AS annual_ret#94371627, CASE WHEN ((std_ret#94371429 = NA) O... [3624255] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624256 [labelType="html" label="MapPartitionsRDD [3624256] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624257 [labelType="html" label="MapPartitionsRDD [3624257] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624258 [labelType="html" label="MapPartitionsRDD [3624258] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302222 { label="WholeStageCodegen (1)"; 3624259 [labelType="html" label="MapPartitionsRDD [3624259] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302221 { label="Exchange"; 3624297 [labelType="html" label="MapPartitionsRDD [3624297] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624251->3624252; 3624252->3624253; 3624253->3624254; 3624254->3624255; 3624255->3624256; 3624256->3624257; 3624257->3624258; 3624258->3624259; 3624259->3624297; }

3624255

digraph G { subgraph clusterstage_512985 { label="Stage 512985"; subgraph cluster6302221 { label="Exchange"; 3624298 [labelType="html" label="ShuffledRowRDD [3624298] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302218 { label="WholeStageCodegen (2)"; 3624299 [labelType="html" label="MapPartitionsRDD [3624299] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302217 { label="InMemoryTableScan"; 3624300 [labelType="html" label="MapPartitionsRDD [3624300] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624301 [labelType="html" label="*(2) Sort [year#94371588 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(year#94371588 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7534732]\n +- *(1) Project [year#94371588, turnover#94371662, (1.0 / cast(turnover#94371662 as double)) AS days_hold#94371704]\n +- *(1) Filter isnotnull(turnover#94371662)\n +- *(1) ColumnarToRow\n +- InMemoryTableScan [turnover#94371662, year#94371588], [isnotnull(turnover#94371662)]\n +- InMemoryRelation [year#94371588, retIC#94371589, resretIC#94371590, numcos#94371624, numdates#94371625, annual_bmret#94371626, annual_ret#94371627, std_ret#94371628, Sharpe_ret#94371630, PctPos_ret#94371632, TR_ret#94371634, IR_ret#94371636, annual_resret#94371638, std_resret#94371640, Sharpe_resret#94371642, PctPos_resret#94371644, TR_resret#94371646, IR_resret#94371648, annual_retnet#94371650, std_retnet#94371652, Sharpe_retnet#94371654, PctPos_retnet#94371656, TR_retnet#94371658, IR_retnet#94371660, turnover#94371662], StorageLevel(disk, ... [3624301] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624302 [labelType="html" label="MapPartitionsRDD [3624302] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624303 [labelType="html" label="MapPartitionsRDD [3624303] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624304 [labelType="html" label="MapPartitionsRDD [3624304] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302214 { label="WholeStageCodegen (1)"; 3624305 [labelType="html" label="MapPartitionsRDD [3624305] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302213 { label="CollectLimit"; 3624306 [labelType="html" label="MapPartitionsRDD [3624306] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624307 [labelType="html" label="MapPartitionsRDD [3624307] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624298->3624299; 3624299->3624300; 3624300->3624301; 3624301->3624302; 3624302->3624303; 3624303->3624304; 3624304->3624305; 3624305->3624306; 3624306->3624307; }

3624297,3624298

3624301

digraph G { subgraph clusterstage_512986 { label="Stage 512986"; subgraph cluster6302213 { label="CollectLimit"; 3624308 [labelType="html" label="ShuffledRowRDD [3624308] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624309 [labelType="html" label="MapPartitionsRDD [3624309] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302355 { label="mapPartitionsInternal"; 3624310 [labelType="html" label="MapPartitionsRDD [3624310] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6302356 { label="map"; 3624311 [labelType="html" label="MapPartitionsRDD [3624311] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624308->3624309; 3624309->3624310; 3624310->3624311; }

3624307,3624308

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512986	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:22:32	12 ms	1/1		1846.0 B
512985	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [year#94371588 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94371588 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7534732] +- (1) Project [year#94371588, turnover#94371662, (1.0 / cast(turnover#94371662 as double)) AS days_hold#94371704] +- (1) Filter isnotnull(turnover#94371662) +- (1) ColumnarToRow +- InMemoryTableScan [turnover#94371662, year#94371588], [isnotnull(turnover#94371662)] +- InMemoryRelation [year#94371588, retIC#94371589, resretIC#94371590, numcos#94371624, numdates#94371625, annual_bmret#94371626, annual_ret#94371627, std_ret#94371628, Sharpe_ret#94371630, PctPos_ret#94371632, TR_ret#94371634, IR_ret#94371636, annual_resret#94371638, std_resret#94371640, Sharpe_resret#94371642, PctPos_resret#94371644, TR_resret#94371646, IR_resret#94371648, annual_retnet#94371650, std_retnet#94371652, Sharpe_retnet#94371654, PctPos_retnet#94371656, TR_retnet#94371658, IR_retnet#94371660, turnover#94371662], StorageLevel(disk, ... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:22:32	0.2 s	26/26		1846.0 B	1846.0 B
512984	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((year#94371422 = NA) OR (year#94371422 = null)) THEN null ELSE cast(year#94371422 as int) END AS year#94371588, CASE WHEN ((retIC#94371423 = NA) OR (retIC#94371423 = null)) THEN null ELSE cast(retIC#94371423 as float) END AS retIC#94371589, CASE WHEN ((resretIC#94371424 = NA) OR (resretIC#94371424 = null)) THEN null ELSE cast(resretIC#94371424 as float) END AS resretIC#94371590, CASE WHEN ((numcos#94371425 = NA) OR (numcos#94371425 = null)) THEN null ELSE cast(numcos#94371425 as float) END AS numcos#94371624, CASE WHEN ((numdates#94371426 = NA) OR (numdates#94371426 = null)) THEN null ELSE cast(numdates#94371426 as int) END AS numdates#94371625, CASE WHEN ((annual_bmret#94371427 = NA) OR (annual_bmret#94371427 = null)) THEN null ELSE cast(annual_bmret#94371427 as float) END AS annual_bmret#94371626, CASE WHEN ((annual_ret#94371428 = NA) OR (annual_ret#94371428 = null)) THEN null ELSE cast(annual_ret#94371428 as float) END AS annual_ret#94371627, CASE WHEN ((std_ret#94371429 = NA) O... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:22:32	14 ms	1/1	4.5 KiB		1846.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

512986

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:22:32

12 ms

1/1

1846.0 B

512985

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(2) Sort [year#94371588 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94371588 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7534732] +- *(1) Project [year#94371588, turnover#94371662, (1.0 / cast(turnover#94371662 as double)) AS days_hold#94371704] +- *(1) Filter isnotnull(turnover#94371662) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94371662, year#94371588], [isnotnull(turnover#94371662)] +- InMemoryRelation [year#94371588, retIC#94371589, resretIC#94371590, numcos#94371624, numdates#94371625, annual_bmret#94371626, annual_ret#94371627, std_ret#94371628, Sharpe_ret#94371630, PctPos_ret#94371632, TR_ret#94371634, IR_ret#94371636, annual_resret#94371638, std_resret#94371640, Sharpe_resret#94371642, PctPos_resret#94371644, TR_resret#94371646, IR_resret#94371648, annual_retnet#94371650, std_retnet#94371652, Sharpe_retnet#94371654, PctPos_retnet#94371656, TR_retnet#94371658, IR_retnet#94371660, turnover#94371662], StorageLevel(disk, ...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:22:32

0.2 s

26/26

1846.0 B

512984

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((year#94371422 = NA) OR (year#94371422 = null)) THEN null ELSE cast(year#94371422 as int) END AS year#94371588, CASE WHEN ((retIC#94371423 = NA) OR (retIC#94371423 = null)) THEN null ELSE cast(retIC#94371423 as float) END AS retIC#94371589, CASE WHEN ((resretIC#94371424 = NA) OR (resretIC#94371424 = null)) THEN null ELSE cast(resretIC#94371424 as float) END AS resretIC#94371590, CASE WHEN ((numcos#94371425 = NA) OR (numcos#94371425 = null)) THEN null ELSE cast(numcos#94371425 as float) END AS numcos#94371624, CASE WHEN ((numdates#94371426 = NA) OR (numdates#94371426 = null)) THEN null ELSE cast(numdates#94371426 as int) END AS numdates#94371625, CASE WHEN ((annual_bmret#94371427 = NA) OR (annual_bmret#94371427 = null)) THEN null ELSE cast(annual_bmret#94371427 as float) END AS annual_bmret#94371626, CASE WHEN ((annual_ret#94371428 = NA) OR (annual_ret#94371428 = null)) THEN null ELSE cast(annual_ret#94371428 as float) END AS annual_ret#94371627, CASE WHEN ((std_ret#94371429 = NA) O...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:22:32

14 ms

1/1

4.5 KiB

1846.0 B

Page: