PlusAMP App Middleware - Details for Job 330573

Status: SUCCEEDED
Submitted: 2025/08/01 18:35:36
Duration: 1 s
Completed Stages: 2

Enable zooming

digraph G { subgraph clusterstage_511689 { label="Stage 511689"; subgraph cluster6278115 { label="Scan csv "; 3610385 [labelType="html" label="FileScanRDD [3610385] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610386 [labelType="html" label="MapPartitionsRDD [3610386] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278112 { label="WholeStageCodegen (1)"; 3610387 [labelType="html" label="MapPartitionsRDD [3610387] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278111 { label="InMemoryTableScan"; 3610388 [labelType="html" label="MapPartitionsRDD [3610388] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610389 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(numcos#94005085 as float) END AS numcos#94005116]\n+- FileScan csv [fractile#94005081,overall#94005082,annual_ret#94005083,annual_resret#94005084,numcos#94005085] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: ... [3610389] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610390 [labelType="html" label="MapPartitionsRDD [3610390] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610391 [labelType="html" label="MapPartitionsRDD [3610391] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610392 [labelType="html" label="MapPartitionsRDD [3610392] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278108 { label="WholeStageCodegen (1)"; 3610393 [labelType="html" label="MapPartitionsRDD [3610393] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278125 { label="Scan csv "; 3610394 [labelType="html" label="FileScanRDD [3610394] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610395 [labelType="html" label="MapPartitionsRDD [3610395] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278122 { label="WholeStageCodegen (1)"; 3610396 [labelType="html" label="MapPartitionsRDD [3610396] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278121 { label="InMemoryTableScan"; 3610397 [labelType="html" label="MapPartitionsRDD [3610397] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610398 [labelType="html" label="*(1) Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105]\n+- FileScan csv [fractile#94005058,cap#94005059,annual_ret#94005060,annual_resret#94005061,numcos#94005062] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters:... [3610398] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610399 [labelType="html" label="MapPartitionsRDD [3610399] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610400 [labelType="html" label="MapPartitionsRDD [3610400] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610401 [labelType="html" label="MapPartitionsRDD [3610401] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278116 { label="WholeStageCodegen (2)"; 3610402 [labelType="html" label="MapPartitionsRDD [3610402] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278107 { label="Union"; 3610403 [labelType="html" label="UnionRDD [3610403] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6278106 { label="InMemoryTableScan"; 3610404 [labelType="html" label="MapPartitionsRDD [3610404] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3610405 [labelType="html" label="Union\n:- *(1) ColumnarToRow\n: +- InMemoryTableScan [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116]\n: +- InMemoryRelation [fractile#94005106, overall#94005107, annual_ret#94005109, annual_resret#94005115, numcos#94005116], StorageLevel(disk, memory, deserialized, 1 replicas)\n: +- *(1) Project [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(num... [3610405] [Cached] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286333 { label="InMemoryTableScan"; 3615114 [labelType="html" label="MapPartitionsRDD [3615114] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615133 [labelType="html" label="MapPartitionsRDD [3615133] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615134 [labelType="html" label="MapPartitionsRDD [3615134] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286330 { label="WholeStageCodegen (1)"; 3615140 [labelType="html" label="MapPartitionsRDD [3615140] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286329 { label="CollectLimit"; 3615141 [labelType="html" label="MapPartitionsRDD [3615141] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615142 [labelType="html" label="MapPartitionsRDD [3615142] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3610385->3610386; 3610386->3610387; 3610387->3610388; 3610388->3610389; 3610389->3610390; 3610390->3610391; 3610391->3610392; 3610392->3610393; 3610394->3610395; 3610395->3610396; 3610396->3610397; 3610397->3610398; 3610398->3610399; 3610399->3610400; 3610400->3610401; 3610401->3610402; 3610393->3610403; 3610402->3610403; 3610403->3610404; 3610404->3610405; 3610405->3615114; 3615114->3615133; 3615133->3615134; 3615134->3615140; 3615140->3615141; 3615141->3615142; }

3610389

3610398

3610405

digraph G { subgraph clusterstage_511690 { label="Stage 511690"; subgraph cluster6286329 { label="CollectLimit"; 3615143 [labelType="html" label="ShuffledRowRDD [3615143] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; 3615145 [labelType="html" label="MapPartitionsRDD [3615145] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286349 { label="mapPartitionsInternal"; 3615146 [labelType="html" label="MapPartitionsRDD [3615146] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } subgraph cluster6286350 { label="map"; 3615147 [labelType="html" label="MapPartitionsRDD [3615147] [Unordered] toLocalIterator at SparkDataStreamBuilder.scala:39"]; } } 3615143->3615145; 3615145->3615146; 3615146->3615147; }

3615142,3615143

Completed Stages (2)

Page:

Stage Id ▾	Pool Name	Description	Submitted	Duration	Tasks: Succeeded/Total	Input	Output	Shuffle Read	Shuffle Write
511690	default	toStream at SparkDataStreamBuilder.scala:39 +details scala.collection.AbstractIterator.toStream(Iterator.scala:1431) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:35:37	10 ms	1/1			281.0 B
511689	default	toLocalIterator at SparkDataStreamBuilder.scala:39 +details RDD: (1) Project [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(numcos#94005085 as float) END AS numcos#94005116] +- FileScan csv [fractile#94005081,overall#94005082,annual_ret#94005083,annual_resret#94005084,numcos#94005085] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: ...(1) Project [CASE WHEN ((fractile#94005058 = NA) OR (fractile#94005058 = null)) THEN null ELSE cast(fractile#94005058 as float) END AS fractile#94005101, CASE WHEN ((cap#94005059 = NA) OR (cap#94005059 = null)) THEN null ELSE cast(cap#94005059 as float) END AS cap#94005102, CASE WHEN ((annual_ret#94005060 = NA) OR (annual_ret#94005060 = null)) THEN null ELSE cast(annual_ret#94005060 as float) END AS annual_ret#94005103, CASE WHEN ((annual_resret#94005061 = NA) OR (annual_resret#94005061 = null)) THEN null ELSE cast(annual_resret#94005061 as float) END AS annual_resret#94005104, CASE WHEN ((numcos#94005062 = NA) OR (numcos#94005062 = null)) THEN null ELSE cast(numcos#94005062 as float) END AS numcos#94005105] +- FileScan csv [fractile#94005058,cap#94005059,annual_ret#94005060,annual_resret#94005061,numcos#94005062] Batched: false, DataFilters: [], Format: CSV, Location: InMemoryFileIndex(1 paths)[file:/srv/plusamp/data/default/ea-market/output/rankviews_history/rank..., PartitionFilters: [], PushedFilters:...Union :- (1) ColumnarToRow : +- InMemoryTableScan [fractile#94005106, annual_ret#94005109, annual_resret#94005115, numcos#94005116] : +- InMemoryRelation [fractile#94005106, overall#94005107, annual_ret#94005109, annual_resret#94005115, numcos#94005116], StorageLevel(disk, memory, deserialized, 1 replicas) : +- (1) Project [CASE WHEN ((fractile#94005081 = NA) OR (fractile#94005081 = null)) THEN null ELSE cast(fractile#94005081 as float) END AS fractile#94005106, CASE WHEN ((overall#94005082 = NA) OR (overall#94005082 = null)) THEN null ELSE cast(overall#94005082 as int) END AS overall#94005107, CASE WHEN ((annual_ret#94005083 = NA) OR (annual_ret#94005083 = null)) THEN null ELSE cast(annual_ret#94005083 as float) END AS annual_ret#94005109, CASE WHEN ((annual_resret#94005084 = NA) OR (annual_resret#94005084 = null)) THEN null ELSE cast(annual_resret#94005084 as float) END AS annual_resret#94005115, CASE WHEN ((numcos#94005085 = NA) OR (numcos#94005085 = null)) THEN null ELSE cast(num... org.apache.spark.sql.Dataset.toLocalIterator(Dataset.scala:3000) plusamp.middleware.model.core.data.SparkDataStreamBuilder.$anonfun$stream$1(SparkDataStreamBuilder.scala:39) plusamp.scala.util.Profile$.time(Profile.scala:22) plusamp.middleware.model.core.data.SparkDataStreamBuilder.<init>(SparkDataStreamBuilder.scala:39) plusamp.middleware.graphql.datafile.SparkAccessor.$anonfun$retrieveData$3(SparkAccessor.scala:77) scala.util.Success.$anonfun$map$1(Try.scala:255) scala.util.Success.map(Try.scala:213) scala.concurrent.Future.$anonfun$map$1(Future.scala:292) scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:33) scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:33) scala.concurrent.impl.CallbackRunnable.run(Promise.scala:64) java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) java.base/java.lang.Thread.run(Thread.java:829)	2025/08/01 18:35:36	26 ms	2/2	648.0 B			281.0 B

Page: