Environment

Runtime Information

NameValue
Java Home/jre
Java Version17.0.12 (Amazon.com Inc.)
Scala Versionversion 2.12.18

Spark Properties

NameValue
fs.azure.account.key.stgiot83ditqa.blob.core.windows.net6bP21LG3HGMaCZskaZ+1atZdh6A/YMAxlgICImcK18L2pZ4UmEPUoUYMau4QGLUsxXI45JOyxprf+AStN6b15Q==
fs.wasbs.implorg.apache.hadoop.fs.azure.NativeAzureFileSystem
spark.app.idspark-5475ee42ad7d4ea48e9c09e78defaed2
spark.app.initial.jar.urlsfile:/opt/spark/jars/spark.jar
spark.app.nameBSSRM_TRANSFORMATION_RUNNER
spark.app.startTime1750263382679
spark.app.submitTime1750263381079
spark.default.parallelism12
spark.driver.bindAddress10.12.2.44
spark.driver.blockManager.port7079
spark.driver.cores1
spark.driver.extraJavaOptions-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false --add-opens java.base/sun.nio.ch=ALL-UNNAMED -XX:+UseStringDeduplication -XX:+UseG1GC -XX:ReservedCodeCacheSize=100M -XX:MaxMetaspaceSize=256m -XX:CompressedClassSpaceSize=256m -Dconsul.config.path=config/spark-service -Dconsul.host=http://common-consul-server
spark.driver.hostcom-bssrm-app-main-app-34cf7e9783d37026-driver-svc.bems-main.svc
spark.driver.memory4g
spark.driver.port7078
spark.dynamicAllocation.cachedExecutorIdleTimeout150s
spark.dynamicAllocation.enabledtrue
spark.dynamicAllocation.executorIdleTimeout150s
spark.dynamicAllocation.maxExecutors12
spark.dynamicAllocation.minExecutors5
spark.dynamicAllocation.shuffleTracking.enabledtrue
spark.eventLog.enabledtrue
spark.executor.cores1
spark.executor.extraJavaOptions-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false --add-opens java.base/sun.nio.ch=ALL-UNNAMED -XX:+UseStringDeduplication -XX:+UseG1GC -XX:ReservedCodeCacheSize=100M -XX:MaxMetaspaceSize=256m -XX:CompressedClassSpaceSize=256m -Dconsul.config.path=config/spark-service -Dconsul.host=http://common-consul-server
spark.executor.heartbeatInterval30s
spark.executor.iddriver
spark.executor.memory4g
spark.executorEnv.SPARK_DRIVER_POD_IP10.12.2.44
spark.jarslocal:/opt/spark/jars/spark.jar,local:///opt/spark/jars/spark.jar
spark.jars.ivy/tmp/.ivy
spark.kubernetes.authenticate.driver.serviceAccountNamebems-main-fiji-bssrm-service-telemetry-cloud-alarms
spark.kubernetes.container.imageghcr.io/etn-electrical/fiji-bssrm-service-telemetry-cloud-alarms:rc_6.0-04.08-647
spark.kubernetes.container.image.pullPolicyAlways
spark.kubernetes.container.image.pullSecrets*********(redacted)
spark.kubernetes.driver.limit.cores2
spark.kubernetes.driver.pod.namebems-main-fiji-bssrm-service-telemetry-cloud-alarms-driver-noempk
spark.kubernetes.driver.podTemplateFilefile:///entry/driver.yaml
spark.kubernetes.driverEnv.CLUSTERbems-main
spark.kubernetes.executor.hadoopConfigMapNamecom-bssrm-app-main-app-34cf7e9783d37026-hadoop-config
spark.kubernetes.executor.label.executor-nametelemetry-cloud-alarms-executor
spark.kubernetes.executor.label.typespark-service
spark.kubernetes.executor.limit.cores2
spark.kubernetes.executor.podNamePrefixbems-main-fiji-bssrm-service-telemetry-cloud-alarms
spark.kubernetes.executor.podTemplateFile/opt/spark/pod-template/pod-spec-template.yml
spark.kubernetes.memoryOverheadFactor0.2
spark.kubernetes.namespacebems-main
spark.kubernetes.resource.typejava
spark.kubernetes.submitInDrivertrue
spark.masterk8s://https://kubernetes.default.svc.cluster.local:443
spark.memory.fraction0.1
spark.memory.storageFraction0.1
spark.network.timeout100s
spark.repl.local.jarslocal:///opt/spark/jars/spark.jar
spark.scheduler.modeFAIR
spark.sql.catalogImplementationin-memory
spark.sql.shuffle.partitions200
spark.sql.streaming.checkpointLocationwasbs://stgiot83qa@stgiot83ditqa.blob.core.windows.net/Checkpoint/bems-main-r2
spark.sql.streaming.forceDeleteTempCheckpointLocationtrue
spark.submit.deployModeclient
spark.submit.pyFiles

Resource Profiles

Resource Profile IdResource Profile Contents
0
Executor Reqs:
	cores: [amount: 1]
	memory: [amount: 4096]
	offHeap: [amount: 0]
Task Reqs:
	cpus: [amount: 1.0]

Hadoop Properties

System Properties

Metrics Properties

Classpath Entries