PlusAMP App Middleware - Details for Job 330560

Status: SUCCEEDED
Submitted: 2025/08/01 16:00:17
Duration: 0.1 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_511664 { label="Stage 511664"; subgraph cluster6286026 { label="Scan csv "; 3614943 [labelType="html" label="FileScanRDD [3614943] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614944 [labelType="html" label="MapPartitionsRDD [3614944] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285988 { label="WholeStageCodegen (1)"; 3614945 [labelType="html" label="MapPartitionsRDD [3614945] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285987 { label="InMemoryTableScan"; 3614946 [labelType="html" label="MapPartitionsRDD [3614946] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614947 [labelType="html" label="*(1) Project [CASE WHEN ((year#94128778 = NA) OR (year#94128778 = null)) THEN null ELSE cast(year#94128778 as float) END AS year#94129013, CASE WHEN ((retIC#94128779 = NA) OR (retIC#94128779 = null)) THEN null ELSE cast(retIC#94128779 as float) END AS retIC#94129015, CASE WHEN ((resretIC#94128780 = NA) OR (resretIC#94128780 = null)) THEN null ELSE cast(resretIC#94128780 as float) END AS resretIC#94129017, CASE WHEN ((numcos#94128781 = NA) OR (numcos#94128781 = null)) THEN null ELSE cast(numcos#94128781 as float) END AS numcos#94129020, CASE WHEN ((numdates#94128782 = NA) OR (numdates#94128782 = null)) THEN null ELSE cast(numdates#94128782 as int) END AS numdates#94129022, CASE WHEN ((annual_bmret#94128783 = NA) OR (annual_bmret#94128783 = null)) THEN null ELSE cast(annual_bmret#94128783 as float) END AS annual_bmret#94129024, CASE WHEN ((annual_ret#94128784 = NA) OR (annual_ret#94128784 = null)) THEN null ELSE cast(annual_ret#94128784 as float) END AS annual_ret#94129026, CASE WHEN ((std_ret#94128785 = NA)... [3614947] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614948 [labelType="html" label="MapPartitionsRDD [3614948] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614949 [labelType="html" label="MapPartitionsRDD [3614949] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614950 [labelType="html" label="MapPartitionsRDD [3614950] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285982 { label="WholeStageCodegen (1)"; 3614951 [labelType="html" label="MapPartitionsRDD [3614951] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285981 { label="Exchange"; 3614975 [labelType="html" label="MapPartitionsRDD [3614975] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3614943->3614944; 3614944->3614945; 3614945->3614946; 3614946->3614947; 3614947->3614948; 3614948->3614949; 3614949->3614950; 3614950->3614951; 3614951->3614975; }

3614947

digraph G { subgraph clusterstage_511665 { label="Stage 511665"; subgraph cluster6285981 { label="Exchange"; 3614976 [labelType="html" label="ShuffledRowRDD [3614976] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285978 { label="WholeStageCodegen (2)"; 3614978 [labelType="html" label="MapPartitionsRDD [3614978] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285977 { label="InMemoryTableScan"; 3614980 [labelType="html" label="MapPartitionsRDD [3614980] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614981 [labelType="html" label="*(2) Sort [year#94129013 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(year#94129013 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7515398]\n +- *(1) Project [year#94129013, turnover#94129069, (1.0 / cast(turnover#94129069 as double)) AS days_hold#94129128]\n +- *(1) Filter isnotnull(turnover#94129069)\n +- *(1) ColumnarToRow\n +- InMemoryTableScan [turnover#94129069, year#94129013], [isnotnull(turnover#94129069)]\n +- InMemoryRelation [year#94129013, retIC#94129015, resretIC#94129017, numcos#94129020, numdates#94129022, annual_bmret#94129024, annual_ret#94129026, std_ret#94129029, Sharpe_ret#94129031, PctPos_ret#94129033, TR_ret#94129035, IR_ret#94129038, annual_resret#94129040, std_resret#94129043, Sharpe_resret#94129045, PctPos_resret#94129047, TR_resret#94129050, IR_resret#94129052, annual_retnet#94129054, std_retnet#94129057, Sharpe_retnet#94129059, PctPos_retnet#94129061, TR_retnet#94129064, IR_retnet#94129066, turnover#94129069], StorageLevel(disk, ... [3614981] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614982 [labelType="html" label="MapPartitionsRDD [3614982] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614983 [labelType="html" label="MapPartitionsRDD [3614983] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614996 [labelType="html" label="MapPartitionsRDD [3614996] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285974 { label="WholeStageCodegen (1)"; 3614997 [labelType="html" label="MapPartitionsRDD [3614997] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6285973 { label="CollectLimit"; 3614998 [labelType="html" label="MapPartitionsRDD [3614998] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3614999 [labelType="html" label="MapPartitionsRDD [3614999] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3614976->3614978; 3614978->3614980; 3614980->3614981; 3614981->3614982; 3614982->3614983; 3614983->3614996; 3614996->3614997; 3614997->3614998; 3614998->3614999; }

3614975,3614976

3614981

digraph G { subgraph clusterstage_511666 { label="Stage 511666"; subgraph cluster6285973 { label="CollectLimit"; 3615000 [labelType="html" label="ShuffledRowRDD [3615000] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615001 [labelType="html" label="MapPartitionsRDD [3615001] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286096 { label="mapPartitionsInternal"; 3615002 [labelType="html" label="MapPartitionsRDD [3615002] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286097 { label="map"; 3615003 [labelType="html" label="MapPartitionsRDD [3615003] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3615000->3615001; 3615001->3615002; 3615002->3615003; }

3614999,3615000

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
511666	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/01 16:00:17	10 ms	1/1		1437.0 B
511665	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [year#94129013 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94129013 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7515398] +- (1) Project [year#94129013, turnover#94129069, (1.0 / cast(turnover#94129069 as double)) AS days_hold#94129128] +- (1) Filter isnotnull(turnover#94129069) +- (1) ColumnarToRow +- InMemoryTableScan [turnover#94129069, year#94129013], [isnotnull(turnover#94129069)] +- InMemoryRelation [year#94129013, retIC#94129015, resretIC#94129017, numcos#94129020, numdates#94129022, annual_bmret#94129024, annual_ret#94129026, std_ret#94129029, Sharpe_ret#94129031, PctPos_ret#94129033, TR_ret#94129035, IR_ret#94129038, annual_resret#94129040, std_resret#94129043, Sharpe_resret#94129045, PctPos_resret#94129047, TR_resret#94129050, IR_resret#94129052, annual_retnet#94129054, std_retnet#94129057, Sharpe_retnet#94129059, PctPos_retnet#94129061, TR_retnet#94129064, IR_retnet#94129066, turnover#94129069], StorageLevel(disk, ... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/01 16:00:17	0.1 s	20/20		1437.0 B	1437.0 B
511664	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((year#94128778 = NA) OR (year#94128778 = null)) THEN null ELSE cast(year#94128778 as float) END AS year#94129013, CASE WHEN ((retIC#94128779 = NA) OR (retIC#94128779 = null)) THEN null ELSE cast(retIC#94128779 as float) END AS retIC#94129015, CASE WHEN ((resretIC#94128780 = NA) OR (resretIC#94128780 = null)) THEN null ELSE cast(resretIC#94128780 as float) END AS resretIC#94129017, CASE WHEN ((numcos#94128781 = NA) OR (numcos#94128781 = null)) THEN null ELSE cast(numcos#94128781 as float) END AS numcos#94129020, CASE WHEN ((numdates#94128782 = NA) OR (numdates#94128782 = null)) THEN null ELSE cast(numdates#94128782 as int) END AS numdates#94129022, CASE WHEN ((annual_bmret#94128783 = NA) OR (annual_bmret#94128783 = null)) THEN null ELSE cast(annual_bmret#94128783 as float) END AS annual_bmret#94129024, CASE WHEN ((annual_ret#94128784 = NA) OR (annual_ret#94128784 = null)) THEN null ELSE cast(annual_ret#94128784 as float) END AS annual_ret#94129026, CASE WHEN ((std_ret#94128785 = NA)... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/01 16:00:17	14 ms	1/1	4.0 KiB		1437.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

511666

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/01 16:00:17

10 ms

1/1

1437.0 B

511665

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(2) Sort [year#94129013 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94129013 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7515398] +- *(1) Project [year#94129013, turnover#94129069, (1.0 / cast(turnover#94129069 as double)) AS days_hold#94129128] +- *(1) Filter isnotnull(turnover#94129069) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94129069, year#94129013], [isnotnull(turnover#94129069)] +- InMemoryRelation [year#94129013, retIC#94129015, resretIC#94129017, numcos#94129020, numdates#94129022, annual_bmret#94129024, annual_ret#94129026, std_ret#94129029, Sharpe_ret#94129031, PctPos_ret#94129033, TR_ret#94129035, IR_ret#94129038, annual_resret#94129040, std_resret#94129043, Sharpe_resret#94129045, PctPos_resret#94129047, TR_resret#94129050, IR_resret#94129052, annual_retnet#94129054, std_retnet#94129057, Sharpe_retnet#94129059, PctPos_retnet#94129061, TR_retnet#94129064, IR_retnet#94129066, turnover#94129069], StorageLevel(disk, ...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/01 16:00:17

0.1 s

20/20

1437.0 B

511664

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((year#94128778 = NA) OR (year#94128778 = null)) THEN null ELSE cast(year#94128778 as float) END AS year#94129013, CASE WHEN ((retIC#94128779 = NA) OR (retIC#94128779 = null)) THEN null ELSE cast(retIC#94128779 as float) END AS retIC#94129015, CASE WHEN ((resretIC#94128780 = NA) OR (resretIC#94128780 = null)) THEN null ELSE cast(resretIC#94128780 as float) END AS resretIC#94129017, CASE WHEN ((numcos#94128781 = NA) OR (numcos#94128781 = null)) THEN null ELSE cast(numcos#94128781 as float) END AS numcos#94129020, CASE WHEN ((numdates#94128782 = NA) OR (numdates#94128782 = null)) THEN null ELSE cast(numdates#94128782 as int) END AS numdates#94129022, CASE WHEN ((annual_bmret#94128783 = NA) OR (annual_bmret#94128783 = null)) THEN null ELSE cast(annual_bmret#94128783 as float) END AS annual_bmret#94129024, CASE WHEN ((annual_ret#94128784 = NA) OR (annual_ret#94128784 = null)) THEN null ELSE cast(annual_ret#94128784 as float) END AS annual_ret#94129026, CASE WHEN ((std_ret#94128785 = NA)...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/01 16:00:17

14 ms

1/1

4.0 KiB

1437.0 B

Page: