PlusAMP App Middleware - Details for Job 331315

Status: SUCCEEDED
Submitted: 2025/08/02 15:19:05
Duration: 0.1 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_512833 { label="Stage 512833"; subgraph cluster6300400 { label="Scan csv "; 3623171 [labelType="html" label="FileScanRDD [3623171] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623172 [labelType="html" label="MapPartitionsRDD [3623172] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300356 { label="WholeStageCodegen (1)"; 3623173 [labelType="html" label="MapPartitionsRDD [3623173] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300355 { label="InMemoryTableScan"; 3623174 [labelType="html" label="MapPartitionsRDD [3623174] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623175 [labelType="html" label="*(1) Project [CASE WHEN ((year#94342605 = NA) OR (year#94342605 = null)) THEN null ELSE cast(year#94342605 as float) END AS year#94342706, CASE WHEN ((retIC#94342606 = NA) OR (retIC#94342606 = null)) THEN null ELSE cast(retIC#94342606 as float) END AS retIC#94342707, CASE WHEN ((resretIC#94342607 = NA) OR (resretIC#94342607 = null)) THEN null ELSE cast(resretIC#94342607 as float) END AS resretIC#94342708, CASE WHEN ((numcos#94342608 = NA) OR (numcos#94342608 = null)) THEN null ELSE cast(numcos#94342608 as float) END AS numcos#94342709, CASE WHEN ((numdates#94342609 = NA) OR (numdates#94342609 = null)) THEN null ELSE cast(numdates#94342609 as int) END AS numdates#94342710, CASE WHEN ((annual_bmret#94342610 = NA) OR (annual_bmret#94342610 = null)) THEN null ELSE cast(annual_bmret#94342610 as float) END AS annual_bmret#94342711, CASE WHEN ((annual_ret#94342611 = NA) OR (annual_ret#94342611 = null)) THEN null ELSE cast(annual_ret#94342611 as float) END AS annual_ret#94342712, CASE WHEN ((std_ret#94342612 = NA)... [3623175] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623176 [labelType="html" label="MapPartitionsRDD [3623176] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623177 [labelType="html" label="MapPartitionsRDD [3623177] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623178 [labelType="html" label="MapPartitionsRDD [3623178] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300350 { label="WholeStageCodegen (1)"; 3623179 [labelType="html" label="MapPartitionsRDD [3623179] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300349 { label="Exchange"; 3623203 [labelType="html" label="MapPartitionsRDD [3623203] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623171->3623172; 3623172->3623173; 3623173->3623174; 3623174->3623175; 3623175->3623176; 3623176->3623177; 3623177->3623178; 3623178->3623179; 3623179->3623203; }

3623175

digraph G { subgraph clusterstage_512834 { label="Stage 512834"; subgraph cluster6300349 { label="Exchange"; 3623204 [labelType="html" label="ShuffledRowRDD [3623204] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300346 { label="WholeStageCodegen (2)"; 3623205 [labelType="html" label="MapPartitionsRDD [3623205] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300345 { label="InMemoryTableScan"; 3623206 [labelType="html" label="MapPartitionsRDD [3623206] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623207 [labelType="html" label="*(2) Sort [year#94342706 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(year#94342706 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532482]\n +- *(1) Project [year#94342706, turnover#94342830, (1.0 / cast(turnover#94342830 as double)) AS days_hold#94342908]\n +- *(1) Filter isnotnull(turnover#94342830)\n +- *(1) ColumnarToRow\n +- InMemoryTableScan [turnover#94342830, year#94342706], [isnotnull(turnover#94342830)]\n +- InMemoryRelation [year#94342706, retIC#94342707, resretIC#94342708, numcos#94342709, numdates#94342710, annual_bmret#94342711, annual_ret#94342712, std_ret#94342713, Sharpe_ret#94342715, PctPos_ret#94342716, TR_ret#94342717, IR_ret#94342718, annual_resret#94342719, std_resret#94342720, Sharpe_resret#94342754, PctPos_resret#94342755, TR_resret#94342789, IR_resret#94342790, annual_retnet#94342791, std_retnet#94342792, Sharpe_retnet#94342793, PctPos_retnet#94342794, TR_retnet#94342828, IR_retnet#94342829, turnover#94342830], StorageLevel(disk, ... [3623207] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623208 [labelType="html" label="MapPartitionsRDD [3623208] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623209 [labelType="html" label="MapPartitionsRDD [3623209] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623210 [labelType="html" label="MapPartitionsRDD [3623210] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300342 { label="WholeStageCodegen (1)"; 3623211 [labelType="html" label="MapPartitionsRDD [3623211] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300341 { label="CollectLimit"; 3623212 [labelType="html" label="MapPartitionsRDD [3623212] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623213 [labelType="html" label="MapPartitionsRDD [3623213] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623204->3623205; 3623205->3623206; 3623206->3623207; 3623207->3623208; 3623208->3623209; 3623209->3623210; 3623210->3623211; 3623211->3623212; 3623212->3623213; }

3623203,3623204

3623207

digraph G { subgraph clusterstage_512835 { label="Stage 512835"; subgraph cluster6300341 { label="CollectLimit"; 3623214 [labelType="html" label="ShuffledRowRDD [3623214] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3623215 [labelType="html" label="MapPartitionsRDD [3623215] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300444 { label="mapPartitionsInternal"; 3623216 [labelType="html" label="MapPartitionsRDD [3623216] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6300445 { label="map"; 3623217 [labelType="html" label="MapPartitionsRDD [3623217] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3623214->3623215; 3623215->3623216; 3623216->3623217; }

3623213,3623214

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
512835	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:19:05	15 ms	1/1		1006.0 B
512834	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [year#94342706 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94342706 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532482] +- (1) Project [year#94342706, turnover#94342830, (1.0 / cast(turnover#94342830 as double)) AS days_hold#94342908] +- (1) Filter isnotnull(turnover#94342830) +- (1) ColumnarToRow +- InMemoryTableScan [turnover#94342830, year#94342706], [isnotnull(turnover#94342830)] +- InMemoryRelation [year#94342706, retIC#94342707, resretIC#94342708, numcos#94342709, numdates#94342710, annual_bmret#94342711, annual_ret#94342712, std_ret#94342713, Sharpe_ret#94342715, PctPos_ret#94342716, TR_ret#94342717, IR_ret#94342718, annual_resret#94342719, std_resret#94342720, Sharpe_resret#94342754, PctPos_resret#94342755, TR_resret#94342789, IR_resret#94342790, annual_retnet#94342791, std_retnet#94342792, Sharpe_retnet#94342793, PctPos_retnet#94342794, TR_retnet#94342828, IR_retnet#94342829, turnover#94342830], StorageLevel(disk, ... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:19:05	69 ms	14/14		1006.0 B	1006.0 B
512833	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((year#94342605 = NA) OR (year#94342605 = null)) THEN null ELSE cast(year#94342605 as float) END AS year#94342706, CASE WHEN ((retIC#94342606 = NA) OR (retIC#94342606 = null)) THEN null ELSE cast(retIC#94342606 as float) END AS retIC#94342707, CASE WHEN ((resretIC#94342607 = NA) OR (resretIC#94342607 = null)) THEN null ELSE cast(resretIC#94342607 as float) END AS resretIC#94342708, CASE WHEN ((numcos#94342608 = NA) OR (numcos#94342608 = null)) THEN null ELSE cast(numcos#94342608 as float) END AS numcos#94342709, CASE WHEN ((numdates#94342609 = NA) OR (numdates#94342609 = null)) THEN null ELSE cast(numdates#94342609 as int) END AS numdates#94342710, CASE WHEN ((annual_bmret#94342610 = NA) OR (annual_bmret#94342610 = null)) THEN null ELSE cast(annual_bmret#94342610 as float) END AS annual_bmret#94342711, CASE WHEN ((annual_ret#94342611 = NA) OR (annual_ret#94342611 = null)) THEN null ELSE cast(annual_ret#94342611 as float) END AS annual_ret#94342712, CASE WHEN ((std_ret#94342612 = NA)... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:19:05	11 ms	1/1	3.4 KiB		1006.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

512835

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:19:05

15 ms

1/1

1006.0 B

512834

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(2) Sort [year#94342706 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94342706 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7532482] +- *(1) Project [year#94342706, turnover#94342830, (1.0 / cast(turnover#94342830 as double)) AS days_hold#94342908] +- *(1) Filter isnotnull(turnover#94342830) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94342830, year#94342706], [isnotnull(turnover#94342830)] +- InMemoryRelation [year#94342706, retIC#94342707, resretIC#94342708, numcos#94342709, numdates#94342710, annual_bmret#94342711, annual_ret#94342712, std_ret#94342713, Sharpe_ret#94342715, PctPos_ret#94342716, TR_ret#94342717, IR_ret#94342718, annual_resret#94342719, std_resret#94342720, Sharpe_resret#94342754, PctPos_resret#94342755, TR_resret#94342789, IR_resret#94342790, annual_retnet#94342791, std_retnet#94342792, Sharpe_retnet#94342793, PctPos_retnet#94342794, TR_retnet#94342828, IR_retnet#94342829, turnover#94342830], StorageLevel(disk, ...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:19:05

69 ms

14/14

1006.0 B

512833

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((year#94342605 = NA) OR (year#94342605 = null)) THEN null ELSE cast(year#94342605 as float) END AS year#94342706, CASE WHEN ((retIC#94342606 = NA) OR (retIC#94342606 = null)) THEN null ELSE cast(retIC#94342606 as float) END AS retIC#94342707, CASE WHEN ((resretIC#94342607 = NA) OR (resretIC#94342607 = null)) THEN null ELSE cast(resretIC#94342607 as float) END AS resretIC#94342708, CASE WHEN ((numcos#94342608 = NA) OR (numcos#94342608 = null)) THEN null ELSE cast(numcos#94342608 as float) END AS numcos#94342709, CASE WHEN ((numdates#94342609 = NA) OR (numdates#94342609 = null)) THEN null ELSE cast(numdates#94342609 as int) END AS numdates#94342710, CASE WHEN ((annual_bmret#94342610 = NA) OR (annual_bmret#94342610 = null)) THEN null ELSE cast(annual_bmret#94342610 as float) END AS annual_bmret#94342711, CASE WHEN ((annual_ret#94342611 = NA) OR (annual_ret#94342611 = null)) THEN null ELSE cast(annual_ret#94342611 as float) END AS annual_ret#94342712, CASE WHEN ((std_ret#94342612 = NA)...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:19:05

11 ms

1/1

3.4 KiB

1006.0 B

Page: