Environment

Runtime Information

NameValue
Java Home/jre
Java Version17.0.12 (Amazon.com Inc.)
Scala Versionversion 2.12.18

Spark Properties

NameValue
fs.azure.account.key.stgiot83ditqa.blob.core.windows.net6bP21LG3HGMaCZskaZ+1atZdh6A/YMAxlgICImcK18L2pZ4UmEPUoUYMau4QGLUsxXI45JOyxprf+AStN6b15Q==
fs.wasbs.implorg.apache.hadoop.fs.azure.NativeAzureFileSystem
spark.app.idspark-230ef6e6201d4aeaa8cc4078c1e3c7d4
spark.app.initial.jar.urlsfile:/opt/spark/jars/spark.jar
spark.app.nameBSSRM_TRANSFORMATION_RUNNER
spark.app.startTime1746476289447
spark.app.submitTime1746476287495
spark.default.parallelism12
spark.driver.bindAddress10.12.0.102
spark.driver.blockManager.port7079
spark.driver.cores1
spark.driver.extraJavaOptions-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false --add-opens java.base/sun.nio.ch=ALL-UNNAMED -XX:+UseStringDeduplication -XX:+UseG1GC -XX:ReservedCodeCacheSize=100M -XX:MaxMetaspaceSize=256m -XX:CompressedClassSpaceSize=256m -Dconsul.config.path=config/spark-service -Dconsul.host=http://common-consul-server
spark.driver.hostcom-bssrm-app-main-app-62cefe96a218dd07-driver-svc.bems-main.svc
spark.driver.memory4g
spark.driver.port7078
spark.dynamicAllocation.cachedExecutorIdleTimeout150s
spark.dynamicAllocation.enabledtrue
spark.dynamicAllocation.executorIdleTimeout150s
spark.dynamicAllocation.maxExecutors12
spark.dynamicAllocation.minExecutors5
spark.dynamicAllocation.shuffleTracking.enabledtrue
spark.eventLog.enabledtrue
spark.executor.cores1
spark.executor.extraJavaOptions-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false --add-opens java.base/sun.nio.ch=ALL-UNNAMED -XX:+UseStringDeduplication -XX:+UseG1GC -XX:ReservedCodeCacheSize=100M -XX:MaxMetaspaceSize=256m -XX:CompressedClassSpaceSize=256m -Dconsul.config.path=config/spark-service -Dconsul.host=http://common-consul-server
spark.executor.heartbeatInterval30s
spark.executor.iddriver
spark.executor.memory4g
spark.executorEnv.SPARK_DRIVER_POD_IP10.12.0.102
spark.jarslocal:/opt/spark/jars/spark.jar,local:///opt/spark/jars/spark.jar
spark.jars.ivy/tmp/.ivy
spark.kubernetes.authenticate.driver.serviceAccountNamebems-main-fiji-bssrm-service-telemetry-cloud-alarms
spark.kubernetes.container.imageghcr.io/etn-electrical/fiji-bssrm-service-telemetry-cloud-alarms:rc_6.0-04.08-647
spark.kubernetes.container.image.pullPolicyAlways
spark.kubernetes.container.image.pullSecrets*********(redacted)
spark.kubernetes.driver.limit.cores2
spark.kubernetes.driver.pod.namebems-main-fiji-bssrm-service-telemetry-cloud-alarms-driver-gqvjto
spark.kubernetes.driver.podTemplateFilefile:///entry/driver.yaml
spark.kubernetes.driverEnv.CLUSTERbems-main
spark.kubernetes.executor.hadoopConfigMapNamecom-bssrm-app-main-app-62cefe96a218dd07-hadoop-config
spark.kubernetes.executor.label.executor-nametelemetry-cloud-alarms-executor
spark.kubernetes.executor.label.typespark-service
spark.kubernetes.executor.limit.cores2
spark.kubernetes.executor.podNamePrefixbems-main-fiji-bssrm-service-telemetry-cloud-alarms
spark.kubernetes.executor.podTemplateFile/opt/spark/pod-template/pod-spec-template.yml
spark.kubernetes.memoryOverheadFactor0.2
spark.kubernetes.namespacebems-main
spark.kubernetes.resource.typejava
spark.kubernetes.submitInDrivertrue
spark.masterk8s://https://kubernetes.default.svc.cluster.local:443
spark.memory.fraction0.1
spark.memory.storageFraction0.1
spark.network.timeout100s
spark.repl.local.jarslocal:///opt/spark/jars/spark.jar
spark.scheduler.modeFAIR
spark.sql.catalogImplementationin-memory
spark.sql.shuffle.partitions200
spark.sql.streaming.checkpointLocationwasbs://stgiot83qa@stgiot83ditqa.blob.core.windows.net/Checkpoint/bems-main-r2
spark.sql.streaming.forceDeleteTempCheckpointLocationtrue
spark.submit.deployModeclient
spark.submit.pyFiles

Resource Profiles

Resource Profile IdResource Profile Contents
0
Executor Reqs:
	cores: [amount: 1]
	memory: [amount: 4096]
	offHeap: [amount: 0]
Task Reqs:
	cpus: [amount: 1.0]

Hadoop Properties

System Properties

Metrics Properties

Classpath Entries