== Physical Plan ==
AdaptiveSparkPlan (11)
+- == Final Plan ==
ResultQueryStage (7), Statistics(sizeInBytes=8.0 EiB)
+- ObjectHashAggregate (6)
+- ShuffleQueryStage (5), Statistics(sizeInBytes=10.6 KiB, rowCount=50)
+- Exchange (4)
+- ObjectHashAggregate (3)
+- * Project (2)
+- * Scan ExistingRDD Delta Table State #12 - hdlfs://332d7051-0d16-44f5-ae76-bf2dcf30365f.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-plant-plant/_delta_log (1)
+- == Initial Plan ==
ObjectHashAggregate (10)
+- Exchange (9)
+- ObjectHashAggregate (8)
+- Project (2)
+- Scan ExistingRDD Delta Table State #12 - hdlfs://332d7051-0d16-44f5-ae76-bf2dcf30365f.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-plant-plant/_delta_log (1)
(1) Scan ExistingRDD Delta Table State #12 - hdlfs://332d7051-0d16-44f5-ae76-bf2dcf30365f.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-plant-plant/_delta_log [codegen id : 1]
Output [10]: [txn#15461652, add#15461653, remove#15461654, metaData#15461655, protocol#15461656, cdc#15461657, checkpointMetadata#15461658, sidecar#15461659, domainMetadata#15461660, commitInfo#15461661]
Arguments: [txn#15461652, add#15461653, remove#15461654, metaData#15461655, protocol#15461656, cdc#15461657, checkpointMetadata#15461658, sidecar#15461659, domainMetadata#15461660, commitInfo#15461661], Delta Table State #12 - hdlfs://332d7051-0d16-44f5-ae76-bf2dcf30365f.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-plant-plant/_delta_log MapPartitionsRDD[885011] at $anonfun$recordDeltaOperationInternal$1 at DatabricksLogging.scala:128, ExistingRDD, UnknownPartitioning(0)
(2) Project [codegen id : 1]
Output [6]: [txn#15461652, add#15461653, remove#15461654, metaData#15461655, protocol#15461656, domainMetadata#15461660]
Input [10]: [txn#15461652, add#15461653, remove#15461654, metaData#15461655, protocol#15461656, cdc#15461657, checkpointMetadata#15461658, sidecar#15461659, domainMetadata#15461660, commitInfo#15461661]
(3) ObjectHashAggregate
Input [6]: [txn#15461652, add#15461653, remove#15461654, metaData#15461655, protocol#15461656, domainMetadata#15461660]
Keys: []
Functions [10]: [partial_collect_set(domainMetadata#15461660, 0, 0), partial_collect_set(txn#15461652, 0, 0), partial_count(protocol#15461656), partial_sum(add#15461653.size), partial_last(metaData#15461655, true), partial_count(metaData#15461655), partial_last(protocol#15461656, true), partial_count(remove#15461654), partial_count(add#15461653), partial_count(txn#15461652)]
Aggregate Attributes [12]: [buf#15461775, buf#15461776, count#15461777L, sum#15461778L, last#15461779, valueSet#15461780, count#15461781L, last#15461782, valueSet#15461783, count#15461784L, count#15461785L, count#15461786L]
Results [12]: [buf#15461787, buf#15461788, count#15461789L, sum#15461790L, last#15461791, valueSet#15461792, count#15461793L, last#15461794, valueSet#15461795, count#15461796L, count#15461797L, count#15461798L]
(4) Exchange
Input [12]: [buf#15461787, buf#15461788, count#15461789L, sum#15461790L, last#15461791, valueSet#15461792, count#15461793L, last#15461794, valueSet#15461795, count#15461796L, count#15461797L, count#15461798L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21683415]
(5) ShuffleQueryStage
Output [12]: [buf#15461787, buf#15461788, count#15461789L, sum#15461790L, last#15461791, valueSet#15461792, count#15461793L, last#15461794, valueSet#15461795, count#15461796L, count#15461797L, count#15461798L]
Arguments: 0
(6) ObjectHashAggregate
Input [12]: [buf#15461787, buf#15461788, count#15461789L, sum#15461790L, last#15461791, valueSet#15461792, count#15461793L, last#15461794, valueSet#15461795, count#15461796L, count#15461797L, count#15461798L]
Keys: []
Functions [10]: [collect_set(domainMetadata#15461660, 0, 0), collect_set(txn#15461652, 0, 0), count(protocol#15461656), sum(add#15461653.size), last(metaData#15461655, true), count(metaData#15461655), last(protocol#15461656, true), count(remove#15461654), count(add#15461653), count(txn#15461652)]
Aggregate Attributes [10]: [collect_set(domainMetadata#15461660, 0, 0)#15461721, collect_set(txn#15461652, 0, 0)#15461722, count(protocol#15461656)#15461723L, sum(add#15461653.size)#15461724L, last(metaData#15461655) ignore nulls#15461725, count(metaData#15461655)#15461726L, last(protocol#15461656) ignore nulls#15461727, count(remove#15461654)#15461728L, count(add#15461653)#15461729L, count(txn#15461652)#15461730L]
Results [14]: [collect_set(domainMetadata#15461660, 0, 0)#15461721 AS domainMetadata#15461706, null AS fileSizeHistogram#15461707, collect_set(txn#15461652, 0, 0)#15461722 AS setTransactions#15461708, null AS numDeletedRecordsOpt#15461709, count(protocol#15461656)#15461723L AS numOfProtocol#15461710L, coalesce(sum(add#15461653.size)#15461724L, 0) AS sizeInBytes#15461711L, last(metaData#15461655) ignore nulls#15461725 AS metadata#15461712, count(metaData#15461655)#15461726L AS numOfMetadata#15461713L, null AS deletedRecordCountsHistogramOpt#15461714, null AS numDeletionVectorsOpt#15461715, last(protocol#15461656) ignore nulls#15461727 AS protocol#15461716, count(remove#15461654)#15461728L AS numOfRemoves#15461717L, count(add#15461653)#15461729L AS numOfFiles#15461718L, count(txn#15461652)#15461730L AS numOfSetTransactions#15461719L]
(7) ResultQueryStage
Output [14]: [domainMetadata#15461706, fileSizeHistogram#15461707, setTransactions#15461708, numDeletedRecordsOpt#15461709, numOfProtocol#15461710L, sizeInBytes#15461711L, metadata#15461712, numOfMetadata#15461713L, deletedRecordCountsHistogramOpt#15461714, numDeletionVectorsOpt#15461715, protocol#15461716, numOfRemoves#15461717L, numOfFiles#15461718L, numOfSetTransactions#15461719L]
Arguments: 1
(8) ObjectHashAggregate
Input [6]: [txn#15461652, add#15461653, remove#15461654, metaData#15461655, protocol#15461656, domainMetadata#15461660]
Keys: []
Functions [10]: [partial_collect_set(domainMetadata#15461660, 0, 0), partial_collect_set(txn#15461652, 0, 0), partial_count(protocol#15461656), partial_sum(add#15461653.size), partial_last(metaData#15461655, true), partial_count(metaData#15461655), partial_last(protocol#15461656, true), partial_count(remove#15461654), partial_count(add#15461653), partial_count(txn#15461652)]
Aggregate Attributes [12]: [buf#15461775, buf#15461776, count#15461777L, sum#15461778L, last#15461779, valueSet#15461780, count#15461781L, last#15461782, valueSet#15461783, count#15461784L, count#15461785L, count#15461786L]
Results [12]: [buf#15461787, buf#15461788, count#15461789L, sum#15461790L, last#15461791, valueSet#15461792, count#15461793L, last#15461794, valueSet#15461795, count#15461796L, count#15461797L, count#15461798L]
(9) Exchange
Input [12]: [buf#15461787, buf#15461788, count#15461789L, sum#15461790L, last#15461791, valueSet#15461792, count#15461793L, last#15461794, valueSet#15461795, count#15461796L, count#15461797L, count#15461798L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=21683404]
(10) ObjectHashAggregate
Input [12]: [buf#15461787, buf#15461788, count#15461789L, sum#15461790L, last#15461791, valueSet#15461792, count#15461793L, last#15461794, valueSet#15461795, count#15461796L, count#15461797L, count#15461798L]
Keys: []
Functions [10]: [collect_set(domainMetadata#15461660, 0, 0), collect_set(txn#15461652, 0, 0), count(protocol#15461656), sum(add#15461653.size), last(metaData#15461655, true), count(metaData#15461655), last(protocol#15461656, true), count(remove#15461654), count(add#15461653), count(txn#15461652)]
Aggregate Attributes [10]: [collect_set(domainMetadata#15461660, 0, 0)#15461721, collect_set(txn#15461652, 0, 0)#15461722, count(protocol#15461656)#15461723L, sum(add#15461653.size)#15461724L, last(metaData#15461655) ignore nulls#15461725, count(metaData#15461655)#15461726L, last(protocol#15461656) ignore nulls#15461727, count(remove#15461654)#15461728L, count(add#15461653)#15461729L, count(txn#15461652)#15461730L]
Results [14]: [collect_set(domainMetadata#15461660, 0, 0)#15461721 AS domainMetadata#15461706, null AS fileSizeHistogram#15461707, collect_set(txn#15461652, 0, 0)#15461722 AS setTransactions#15461708, null AS numDeletedRecordsOpt#15461709, count(protocol#15461656)#15461723L AS numOfProtocol#15461710L, coalesce(sum(add#15461653.size)#15461724L, 0) AS sizeInBytes#15461711L, last(metaData#15461655) ignore nulls#15461725 AS metadata#15461712, count(metaData#15461655)#15461726L AS numOfMetadata#15461713L, null AS deletedRecordCountsHistogramOpt#15461714, null AS numDeletionVectorsOpt#15461715, last(protocol#15461656) ignore nulls#15461727 AS protocol#15461716, count(remove#15461654)#15461728L AS numOfRemoves#15461717L, count(add#15461653)#15461729L AS numOfFiles#15461718L, count(txn#15461652)#15461730L AS numOfSetTransactions#15461719L]
(11) AdaptiveSparkPlan
Output [14]: [domainMetadata#15461706, fileSizeHistogram#15461707, setTransactions#15461708, numDeletedRecordsOpt#15461709, numOfProtocol#15461710L, sizeInBytes#15461711L, metadata#15461712, numOfMetadata#15461713L, deletedRecordCountsHistogramOpt#15461714, numDeletionVectorsOpt#15461715, protocol#15461716, numOfRemoves#15461717L, numOfFiles#15461718L, numOfSetTransactions#15461719L]
Arguments: isFinalPlan=true