PlusAMP App Middleware - Details for Job 331463

Status: SUCCEEDED
Submitted: 2025/08/02 15:24:16
Duration: 0.2 s
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_513058 { label="Stage 513058"; subgraph cluster6303182 { label="Scan csv "; 3624777 [labelType="html" label="FileScanRDD [3624777] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624779 [labelType="html" label="MapPartitionsRDD [3624779] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303151 { label="WholeStageCodegen (1)"; 3624782 [labelType="html" label="MapPartitionsRDD [3624782] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303150 { label="InMemoryTableScan"; 3624783 [labelType="html" label="MapPartitionsRDD [3624783] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624787 [labelType="html" label="*(1) Project [CASE WHEN ((year#94385350 = NA) OR (year#94385350 = null)) THEN null ELSE cast(year#94385350 as int) END AS year#94385518, CASE WHEN ((retIC#94385351 = NA) OR (retIC#94385351 = null)) THEN null ELSE cast(retIC#94385351 as float) END AS retIC#94385520, CASE WHEN ((resretIC#94385352 = NA) OR (resretIC#94385352 = null)) THEN null ELSE cast(resretIC#94385352 as float) END AS resretIC#94385523, CASE WHEN ((numcos#94385353 = NA) OR (numcos#94385353 = null)) THEN null ELSE cast(numcos#94385353 as float) END AS numcos#94385525, CASE WHEN ((numdates#94385354 = NA) OR (numdates#94385354 = null)) THEN null ELSE cast(numdates#94385354 as int) END AS numdates#94385527, CASE WHEN ((annual_bmret#94385355 = NA) OR (annual_bmret#94385355 = null)) THEN null ELSE cast(annual_bmret#94385355 as float) END AS annual_bmret#94385530, CASE WHEN ((annual_ret#94385356 = NA) OR (annual_ret#94385356 = null)) THEN null ELSE cast(annual_ret#94385356 as float) END AS annual_ret#94385532, CASE WHEN ((std_ret#94385357 = NA) O... [3624787] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624789 [labelType="html" label="MapPartitionsRDD [3624789] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624792 [labelType="html" label="MapPartitionsRDD [3624792] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624797 [labelType="html" label="MapPartitionsRDD [3624797] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303145 { label="WholeStageCodegen (1)"; 3624801 [labelType="html" label="MapPartitionsRDD [3624801] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303144 { label="Exchange"; 3624822 [labelType="html" label="MapPartitionsRDD [3624822] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624777->3624779; 3624779->3624782; 3624782->3624783; 3624783->3624787; 3624787->3624789; 3624789->3624792; 3624792->3624797; 3624797->3624801; 3624801->3624822; }

3624787

digraph G { subgraph clusterstage_513059 { label="Stage 513059"; subgraph cluster6303144 { label="Exchange"; 3624823 [labelType="html" label="ShuffledRowRDD [3624823] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303141 { label="WholeStageCodegen (2)"; 3624824 [labelType="html" label="MapPartitionsRDD [3624824] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303140 { label="InMemoryTableScan"; 3624825 [labelType="html" label="MapPartitionsRDD [3624825] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624826 [labelType="html" label="*(2) Sort [year#94385518 ASC NULLS FIRST], true, 0\n+- Exchange rangepartitioning(year#94385518 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535880]\n +- *(1) Project [year#94385518, turnover#94385573, (1.0 / cast(turnover#94385573 as double)) AS days_hold#94385633]\n +- *(1) Filter isnotnull(turnover#94385573)\n +- *(1) ColumnarToRow\n +- InMemoryTableScan [turnover#94385573, year#94385518], [isnotnull(turnover#94385573)]\n +- InMemoryRelation [year#94385518, retIC#94385520, resretIC#94385523, numcos#94385525, numdates#94385527, annual_bmret#94385530, annual_ret#94385532, std_ret#94385534, Sharpe_ret#94385536, PctPos_ret#94385539, TR_ret#94385541, IR_ret#94385543, annual_resret#94385545, std_resret#94385548, Sharpe_resret#94385550, PctPos_resret#94385552, TR_resret#94385555, IR_resret#94385557, annual_retnet#94385559, std_retnet#94385561, Sharpe_retnet#94385563, PctPos_retnet#94385566, TR_retnet#94385568, IR_retnet#94385571, turnover#94385573], StorageLevel(disk, ... [3624826] [Cached] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624827 [labelType="html" label="MapPartitionsRDD [3624827] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624830 [labelType="html" label="MapPartitionsRDD [3624830] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624834 [labelType="html" label="MapPartitionsRDD [3624834] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303137 { label="WholeStageCodegen (1)"; 3624841 [labelType="html" label="MapPartitionsRDD [3624841] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303136 { label="CollectLimit"; 3624842 [labelType="html" label="MapPartitionsRDD [3624842] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624844 [labelType="html" label="MapPartitionsRDD [3624844] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624823->3624824; 3624824->3624825; 3624825->3624826; 3624826->3624827; 3624827->3624830; 3624830->3624834; 3624834->3624841; 3624841->3624842; 3624842->3624844; }

3624822,3624823

3624826

digraph G { subgraph clusterstage_513060 { label="Stage 513060"; subgraph cluster6303136 { label="CollectLimit"; 3624845 [labelType="html" label="ShuffledRowRDD [3624845] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3624846 [labelType="html" label="MapPartitionsRDD [3624846] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303273 { label="mapPartitionsInternal"; 3624847 [labelType="html" label="MapPartitionsRDD [3624847] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6303274 { label="map"; 3624848 [labelType="html" label="MapPartitionsRDD [3624848] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3624845->3624846; 3624846->3624847; 3624847->3624848; }

3624844,3624845

Completed Stages (3)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Shuffle Read	Shuffle Write
513060	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:24:17	10 ms	1/1		1846.0 B
513059	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (2) Sort [year#94385518 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94385518 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535880] +- (1) Project [year#94385518, turnover#94385573, (1.0 / cast(turnover#94385573 as double)) AS days_hold#94385633] +- (1) Filter isnotnull(turnover#94385573) +- (1) ColumnarToRow +- InMemoryTableScan [turnover#94385573, year#94385518], [isnotnull(turnover#94385573)] +- InMemoryRelation [year#94385518, retIC#94385520, resretIC#94385523, numcos#94385525, numdates#94385527, annual_bmret#94385530, annual_ret#94385532, std_ret#94385534, Sharpe_ret#94385536, PctPos_ret#94385539, TR_ret#94385541, IR_ret#94385543, annual_resret#94385545, std_resret#94385548, Sharpe_resret#94385550, PctPos_resret#94385552, TR_resret#94385555, IR_resret#94385557, annual_retnet#94385559, std_retnet#94385561, Sharpe_retnet#94385563, PctPos_retnet#94385566, TR_retnet#94385568, IR_retnet#94385571, turnover#94385573], StorageLevel(disk, ... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:24:16	0.1 s	26/26		1846.0 B	1846.0 B
513058	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: *(1) Project [CASE WHEN ((year#94385350 = NA) OR (year#94385350 = null)) THEN null ELSE cast(year#94385350 as int) END AS year#94385518, CASE WHEN ((retIC#94385351 = NA) OR (retIC#94385351 = null)) THEN null ELSE cast(retIC#94385351 as float) END AS retIC#94385520, CASE WHEN ((resretIC#94385352 = NA) OR (resretIC#94385352 = null)) THEN null ELSE cast(resretIC#94385352 as float) END AS resretIC#94385523, CASE WHEN ((numcos#94385353 = NA) OR (numcos#94385353 = null)) THEN null ELSE cast(numcos#94385353 as float) END AS numcos#94385525, CASE WHEN ((numdates#94385354 = NA) OR (numdates#94385354 = null)) THEN null ELSE cast(numdates#94385354 as int) END AS numdates#94385527, CASE WHEN ((annual_bmret#94385355 = NA) OR (annual_bmret#94385355 = null)) THEN null ELSE cast(annual_bmret#94385355 as float) END AS annual_bmret#94385530, CASE WHEN ((annual_ret#94385356 = NA) OR (annual_ret#94385356 = null)) THEN null ELSE cast(annual_ret#94385356 as float) END AS annual_ret#94385532, CASE WHEN ((std_ret#94385357 = NA) O... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426) java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)	2025/08/02 15:24:16	17 ms	1/1	4.5 KiB		1846.0 B

Stage Id ▾

Pool Name

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

513060

default

toStream at SparkDataStreamBuilder.scala:39 +details

scala.collection.AbstractIterator.toStream(Iterator.scala:1431)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:24:17

10 ms

1/1

1846.0 B

513059

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(2) Sort [year#94385518 ASC NULLS FIRST], true, 0 +- Exchange rangepartitioning(year#94385518 ASC NULLS FIRST, 200), ENSURE_REQUIREMENTS, [id=#7535880] +- *(1) Project [year#94385518, turnover#94385573, (1.0 / cast(turnover#94385573 as double)) AS days_hold#94385633] +- *(1) Filter isnotnull(turnover#94385573) +- *(1) ColumnarToRow +- InMemoryTableScan [turnover#94385573, year#94385518], [isnotnull(turnover#94385573)] +- InMemoryRelation [year#94385518, retIC#94385520, resretIC#94385523, numcos#94385525, numdates#94385527, annual_bmret#94385530, annual_ret#94385532, std_ret#94385534, Sharpe_ret#94385536, PctPos_ret#94385539, TR_ret#94385541, IR_ret#94385543, annual_resret#94385545, std_resret#94385548, Sharpe_resret#94385550, PctPos_resret#94385552, TR_resret#94385555, IR_resret#94385557, annual_retnet#94385559, std_retnet#94385561, Sharpe_retnet#94385563, PctPos_retnet#94385566, TR_retnet#94385568, IR_retnet#94385571, turnover#94385573], StorageLevel(disk, ...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:24:16

0.1 s

26/26

1846.0 B

513058

default

toLocalIterator at SparkDataStreamBuilder.scala:39 +details

RDD: *(1) Project [CASE WHEN ((year#94385350 = NA) OR (year#94385350 = null)) THEN null ELSE cast(year#94385350 as int) END AS year#94385518, CASE WHEN ((retIC#94385351 = NA) OR (retIC#94385351 = null)) THEN null ELSE cast(retIC#94385351 as float) END AS retIC#94385520, CASE WHEN ((resretIC#94385352 = NA) OR (resretIC#94385352 = null)) THEN null ELSE cast(resretIC#94385352 as float) END AS resretIC#94385523, CASE WHEN ((numcos#94385353 = NA) OR (numcos#94385353 = null)) THEN null ELSE cast(numcos#94385353 as float) END AS numcos#94385525, CASE WHEN ((numdates#94385354 = NA) OR (numdates#94385354 = null)) THEN null ELSE cast(numdates#94385354 as int) END AS numdates#94385527, CASE WHEN ((annual_bmret#94385355 = NA) OR (annual_bmret#94385355 = null)) THEN null ELSE cast(annual_bmret#94385355 as float) END AS annual_bmret#94385530, CASE WHEN ((annual_ret#94385356 = NA) OR (annual_ret#94385356 = null)) THEN null ELSE cast(annual_ret#94385356 as float) END AS annual_ret#94385532, CASE WHEN ((std_ret#94385357 = NA) O...

org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39)
plusamp.scala.util.Profile$.time(Profile.scala:22)
plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39)
plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77)
scala.util.Success.$anonfun$map$1(Try.scala:255)
scala.util.Success.map(Try.scala:213)
scala.concurrent.Future.$anonfun$map$1(Future.scala:292)
scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33)
scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33)
scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64)
java.base/java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1426)
java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290)
java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020)
java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656)
java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594)
java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183)

2025/08/02 15:24:16

17 ms

1/1

4.5 KiB

1846.0 B

Page: