Spark On K8s源代码解析
发表于：2024-10-01 作者：千家信息网编辑
千家信息网最后更新 2024年10月01日，resource-managers\pom.xmlorg.apache.spark.deploy.SparkSubmit.scalaorg.apache.spark.deploy.k8s.submit
千家信息网最后更新 2024年10月01日Spark On K8s源代码解析
resource-managers\pom.xml
org.apache.spark.deploy.SparkSubmit.scala
org.apache.spark.deploy.k8s.submit.Client.scalaorg.apache.spark.deploy.k8s.submit.submitsteps.KubernetesDriverSpec.scalaorg.apache.spark.deploy.k8s.submit.Client.scaladebug：
0：
1：
2：
3：
4：
5：
結果：
{ "kind": "Pod", "apiVersion": "v1", "metadata": { "name": "iot-stream-analysis-driver-104", "namespace": "spark-cluster", "selfLink": "/api/v1/namespaces/spark-cluster/pods/iot-stream-analysis-driver-104", "uid": "9897bb77-b4c6-11e8-910e-0050568848e8", "resourceVersion": "32841344", "creationTimestamp": "2018-09-10T06:56:09Z", "labels": { "spark-app-selector": "spark-a680ef1ed8364d3b96dfbc9e5d3f1d83", "spark-role": "driver" }, "annotations": { "spark-app-name": "spark-job-on-k8s-104" } }, "spec": { "volumes": [ { "name": "spark-local-dir-0-spark-1ce8faad-1066-4386-baa0-f8a9de2231f3", "emptyDir": {} }, { "name": "spark-init-properties", "configMap": { "name": "spark-job-on-k8s-104-1536562620604-init-config", "items": [ { "key": "download-submitted-files", "path": "spark-init.properties" } ], "defaultMode": 420 } }, { "name": "download-jars-volume", "emptyDir": {} }, { "name": "download-files", "emptyDir": {} }, { "name": "spark-init-secret", "secret": { "secretName": "spark-job-on-k8s-104-1536562620604-init-secret", "defaultMode": 420 } }, { "name": "spark-token-kpkrp", "secret": { "secretName": "spark-token-kpkrp", "defaultMode": 420 } } ], "initContainers": [ { "name": "spark-init", "image": "registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1", "args": [ "/etc/spark-init/spark-init.properties" ], "resources": {}, "volumeMounts": [ { "name": "spark-init-properties", "mountPath": "/etc/spark-init" }, { "name": "download-jars-volume", "mountPath": "/var/spark-data/spark-jars" }, { "name": "download-files", "mountPath": "/var/spark-data/spark-files" }, { "name": "spark-init-secret", "mountPath": "/mnt/secrets/spark-init" }, { "name": "spark-token-kpkrp", "readOnly": true, "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" } ], "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "imagePullPolicy": "Always" } ], "containers": [ { "name": "spark-kubernetes-driver", "image": "registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1", "env": [ { "name": "SPARK_DRIVER_MEMORY", "value": "1024m" }, { "name": "SPARK_DRIVER_CLASS", "value": "com.inspur.iot.RuleEngine" }, { "name": "SPARK_DRIVER_ARGS", "value": "--base64=true --rule=c2VsZWN0IHRpbWVTdGFtcCBBcyBrZXksIGNvbmNhdF93cygifCIsIHN0YXRlLnJlcG9ydGVkLnRlbXBlcmF0dXJlLCBjbGllbnRUb2tlbikgYXMgdmFsdWUgZnJvbSB0b3BpY3M= --sample={\"timeStamp\":1531381822,\"clientToken\":\"clientId_lamp\",\"state\":{\"reported\":{\"temperature\":23}}} --source-type=kafka --source={\"kafka.bootstrap.servers\":\"insight-kafka-svc.default:9092\",\"subscribe\":\"sensor\"} --sink-type=console --verbose" }, { "name": "SPARK_DRIVER_BIND_ADDRESS", "valueFrom": { "fieldRef": { "apiVersion": "v1", "fieldPath": "status.podIP" } } }, { "name": "SPARK_MOUNTED_CLASSPATH", "value": "/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar" }, { "name": "SPARK_MOUNTED_FILES_DIR", "value": "/var/spark-data/spark-files" }, { "name": "SPARK_JAVA_OPT_0", "value": "-Dspark.kubernetes.svc.domain=svc.dev161" }, { "name": "SPARK_JAVA_OPT_1", "value": "-Dspark.kubernetes.initcontainer.executor.configmapkey=download-submitted-files" }, { "name": "SPARK_JAVA_OPT_2", "value": "-Dspark.app.id=spark-a680ef1ed8364d3b96dfbc9e5d3f1d83" }, { "name": "SPARK_JAVA_OPT_3", "value": "-Dspark.submit.deployMode=cluster" }, { "name": "SPARK_JAVA_OPT_4", "value": "-Dspark.kubernetes.authenticate.driver.serviceAccountName=spark" }, { "name": "SPARK_JAVA_OPT_5", "value": "-Dspark.kubernetes.resourceStagingServer.uri=http://10.110.20.161:31000" }, { "name": "SPARK_JAVA_OPT_6", "value": "-Dspark.app.name=spark-job-on-k8s-104" }, { "name": "SPARK_JAVA_OPT_7", "value": "-Dspark.jars=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar" }, { "name": "SPARK_JAVA_OPT_8", "value": "-Dspark.driver.host=spark-job-on-k8s-104-1536562620604-driver-svc.spark-cluster.svc.cluster.local" }, { "name": "SPARK_JAVA_OPT_9", "value": "-Dspark.local.dir=/mnt/tmp/spark-local/spark-1ce8faad-1066-4386-baa0-f8a9de2231f3" }, { "name": "SPARK_JAVA_OPT_10", "value": "-Dspark.kubernetes.initcontainer.docker.image=registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1" }, { "name": "SPARK_JAVA_OPT_11", "value": "-Dspark.kubernetes.initcontainer.executor.configmapname=spark-job-on-k8s-104-1536562620604-init-config" }, { "name": "SPARK_JAVA_OPT_12", "value": "-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.name=spark-job-on-k8s-104-1536562620604-init-secret" }, { "name": "SPARK_JAVA_OPT_13", "value": "-Dspark.driver.cores=1" }, { "name": "SPARK_JAVA_OPT_14", "value": "-Dspark.kubernetes.docker.image.pullPolicy=Always" }, { "name": "SPARK_JAVA_OPT_15", "value": "-Dspark.kubernetes.driver.docker.image=registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1" }, { "name": "SPARK_JAVA_OPT_16", "value": "-Dspark.kubernetes.driver.nodeport=31202" }, { "name": "SPARK_JAVA_OPT_17", "value": "-Dspark.driver.blockManager.port=7079" }, { "name": "SPARK_JAVA_OPT_18", "value": "-Dspark.kubernetes.namespace=spark-cluster" }, { "name": "SPARK_JAVA_OPT_19", "value": "-Dspark.kubernetes.driver.pod.name=iot-stream-analysis-driver-104" }, { "name": "SPARK_JAVA_OPT_20", "value": "-Dspark.driver.memory=1024m" }, { "name": "SPARK_JAVA_OPT_21", "value": "-Dspark.kubernetes.submission.waitAppCompletion=true" }, { "name": "SPARK_JAVA_OPT_22", "value": "-Dspark.driver.port=7078" }, { "name": "SPARK_JAVA_OPT_23", "value": "-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.mountDir=/mnt/secrets/spark-init" }, { "name": "SPARK_JAVA_OPT_24", "value": "-Dspark.master=k8s://http://10.110.20.161:8080" }, { "name": "SPARK_JAVA_OPT_25", "value": "-Dspark.executor.memory=1024m" }, { "name": "SPARK_JAVA_OPT_26", "value": "-Dspark.kubernetes.executor.podNamePrefix=spark-job-on-k8s-104-1536562620604" }, { "name": "SPARK_JAVA_OPT_27", "value": "-Dspark.ui.enabled=false" }, { "name": "SPARK_JAVA_OPT_28", "value": "-Dspark.executor.instances=1" }, { "name": "SPARK_JAVA_OPT_29", "value": "-Dspark.kubernetes.executor.docker.image=registry.dev161.com:5000/iot/spark-executor:v2.2.0-kubernetes-0.5.1" }, { "name": "SPARK_JAVA_OPT_30", "value": "-Dspark.executor.cores=1" } ], "resources": { "limits": { "memory": "1408Mi" }, "requests": { "cpu": "1", "memory": "1Gi" } }, "volumeMounts": [ { "name": "spark-local-dir-0-spark-1ce8faad-1066-4386-baa0-f8a9de2231f3", "mountPath": "/mnt/tmp/spark-local/spark-1ce8faad-1066-4386-baa0-f8a9de2231f3" }, { "name": "download-jars-volume", "mountPath": "/var/spark-data/spark-jars" }, { "name": "download-files", "mountPath": "/var/spark-data/spark-files" }, { "name": "spark-token-kpkrp", "readOnly": true, "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" } ], "terminationMessagePath": "/dev/termination-log", "terminationMessagePolicy": "File", "imagePullPolicy": "Always" } ], "restartPolicy": "Never", "terminationGracePeriodSeconds": 30, "dnsPolicy": "ClusterFirst", "serviceAccountName": "spark", "serviceAccount": "spark", "nodeName": "master1", "securityContext": {}, "schedulerName": "default-scheduler" }, "status": { "phase": "Running", "conditions": [ { "type": "Initialized", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-09-10T06:56:18Z" }, { "type": "Ready", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-09-10T06:56:19Z" }, { "type": "PodScheduled", "status": "True", "lastProbeTime": null, "lastTransitionTime": "2018-09-10T06:56:09Z" } ], "hostIP": "10.110.20.161", "podIP": "161.161.104.93", "startTime": "2018-09-10T06:56:09Z", "initContainerStatuses": [ { "name": "spark-init", "state": { "terminated": { "exitCode": 0, "reason": "Completed", "startedAt": "2018-09-10T06:56:13Z", "finishedAt": "2018-09-10T06:56:18Z", "containerID": "docker://c127670c8e9c1157b00ce29145dff658a89576dc3d785355c6f4dce00df1606f" } }, "lastState": {}, "ready": true, "restartCount": 0, "image": "registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1", "imageID": "docker-pullable://registry.dev161.com:5000/iot/spark-init@sha256:adca6ed85868d821e3f63e49b7d44765954bcf5e4131a4d419cd9f4dc2235401", "containerID": "docker://c127670c8e9c1157b00ce29145dff658a89576dc3d785355c6f4dce00df1606f" } ], "containerStatuses": [ { "name": "spark-kubernetes-driver", "state": { "running": { "startedAt": "2018-09-10T06:56:19Z" } }, "lastState": {}, "ready": true, "restartCount": 0, "image": "registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1", "imageID": "docker-pullable://registry.dev161.com:5000/iot/spark-driver@sha256:6cf4ab9fd7ff1c21ddd1d66feb76107928c844cfaae912172bc618734942ff80", "containerID": "docker://6cb1f8004505aa3440827691720bc35eff1a64936da8fd4d71e82afade29e05d" } ], "qosClass": "Burstable" }}{ "kind": "ConfigMap", "apiVersion": "v1", "metadata": { "name": "spark-job-on-k8s-104-1536562620604-init-config", "namespace": "spark-cluster", "selfLink": "/api/v1/namespaces/spark-cluster/configmaps/spark-job-on-k8s-104-1536562620604-init-config", "uid": "99991bea-b4c6-11e8-910e-0050568848e8", "resourceVersion": "32841315", "creationTimestamp": "2018-09-10T06:56:10Z", "ownerReferences": [ { "apiVersion": "v1", "kind": "Pod", "name": "iot-stream-analysis-driver-104", "uid": "9897bb77-b4c6-11e8-910e-0050568848e8", "controller": true } ] }, "data": { "download-submitted-files": "#Java properties built from Kubernetes config map with name: spark-job-on-k8s-104-1536562620604-init-config and config map key: $configMapKey\r\n#Mon Sep 10 14:57:08 CST 2018\r\nspark.kubernetes.initcontainer.downloadJarsSecretLocation=/mnt/secrets/spark-init/downloadSubmittedJarsSecret\r\nspark.kubernetes.initcontainer.downloadFilesSecretLocation=/mnt/secrets/spark-init/downloadSubmittedFilesSecret\r\nspark.kubernetes.initcontainer.downloadJarsResourceIdentifier=a6c0295f-495a-4967-856e-7e2e607cb0f8\r\nspark.kubernetes.initcontainer.downloadFilesResourceIdentifier=d673b9f6-acd0-4cbd-9851-fc5a9ae1c760\r\nspark.ssl.kubernetes.resourceStagingServer.enabled=false\r\nspark.kubernetes.resourceStagingServer.uri=http\\://10.110.20.161\\:31000\r\nspark.kubernetes.mountdependencies.filesDownloadDir=/var/spark-data/spark-files\r\nspark.kubernetes.mountdependencies.jarsDownloadDir=/var/spark-data/spark-jars\r\n" }}{ "kind": "Service", "apiVersion": "v1", "metadata": { "name": "spark-job-on-k8s-104-1536562620604-driver-svc", "namespace": "spark-cluster", "selfLink": "/api/v1/namespaces/spark-cluster/services/spark-job-on-k8s-104-1536562620604-driver-svc", "uid": "99892978-b4c6-11e8-910e-0050568848e8", "resourceVersion": "32841312", "creationTimestamp": "2018-09-10T06:56:10Z", "ownerReferences": [ { "apiVersion": "v1", "kind": "Pod", "name": "iot-stream-analysis-driver-104", "uid": "9897bb77-b4c6-11e8-910e-0050568848e8", "controller": true } ] }, "spec": { "ports": [ { "name": "driver-rpc-port", "protocol": "TCP", "port": 7078, "targetPort": 7078 }, { "name": "blockmanager", "protocol": "TCP", "port": 7079, "targetPort": 7079 } ], "selector": { "spark-app-selector": "spark-a680ef1ed8364d3b96dfbc9e5d3f1d83", "spark-role": "driver" }, "clusterIP": "None", "type": "ClusterIP", "sessionAffinity": "None" }, "status": { "loadBalancer": {} }}container:
init-container:
[ { "Id": "d11c77e3981cd932284d5503f3ea2450a5ae65736cd34d7daa264f33d45b0957", "Created": "2018-09-11T03:47:40.443123953Z", "Path": "/opt/entrypoint.sh", "Args": [ "/opt/spark/bin/spark-class", "org.apache.spark.deploy.rest.k8s.KubernetesSparkDependencyDownloadInitContainer", "/etc/spark-init/spark-init.properties" ], "State": { "Status": "exited", "Running": false, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 0, "ExitCode": 0, "Error": "", "StartedAt": "2018-09-11T03:47:40.925573911Z", "FinishedAt": "2018-09-11T03:47:45.500154065Z" }, "Image": "sha256:45b585586ede8839199405d7cb01fede31f5dc9fdeda7d7cf4b0aaa24050b4db", "ResolvConfPath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/resolv.conf", "HostnamePath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/hostname", "HostsPath": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts", "LogPath": "/var/lib/docker/containers/d11c77e3981cd932284d5503f3ea2450a5ae65736cd34d7daa264f33d45b0957/d11c77e3981cd932284d5503f3ea2450a5ae65736cd34d7daa264f33d45b0957-json.log", "Name": "/k8s_spark-init_iot-stream-analysis-driver-100_spark-cluster_64491f22-b575-11e8-910e-0050568848e8_0", "RestartCount": 0, "Driver": "overlay", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [ "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~configmap/spark-init-properties:/etc/spark-init", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-init-secret:/mnt/secrets/spark-init", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp:/var/run/secrets/kubernetes.io/serviceaccount:ro", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-init/206592cf:/dev/termination-log" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": { "max-file": "5", "max-size": "50m" } }, "NetworkMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b", "PortBindings": null, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b", "Cgroup": "", "Links": null, "OomScoreAdj": 999, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": [ "seccomp=unconfined" ], "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 2, "Memory": 0, "NanoCpus": 0, "CgroupParent": "kubepods-burstable-pod64491f22_b575_11e8_910e_0050568848e8.slice", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": [], "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 0, "MemorySwappiness": -1, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0 }, "GraphDriver": { "Name": "overlay", "Data": { "LowerDir": "/var/lib/docker/overlay/24dfa74479a001ecabd6d8ab9f61337a898ad9ab78e914b5ab17096c357eaf5e/root", "MergedDir": "/var/lib/docker/overlay/c31c6eec68c6405dd62c8dd7dfea4707cebca534df2dced5f0b158969c0dd7e6/merged", "UpperDir": "/var/lib/docker/overlay/c31c6eec68c6405dd62c8dd7dfea4707cebca534df2dced5f0b158969c0dd7e6/upper", "WorkDir": "/var/lib/docker/overlay/c31c6eec68c6405dd62c8dd7dfea4707cebca534df2dced5f0b158969c0dd7e6/work" } }, "Mounts": [ { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts", "Destination": "/etc/hosts", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-init/206592cf", "Destination": "/dev/termination-log", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~configmap/spark-init-properties", "Destination": "/etc/spark-init", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume", "Destination": "/var/spark-data/spark-jars", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files", "Destination": "/var/spark-data/spark-files", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-init-secret", "Destination": "/mnt/secrets/spark-init", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp", "Destination": "/var/run/secrets/kubernetes.io/serviceaccount", "Mode": "ro", "RW": false, "Propagation": "" } ], "Config": { "Hostname": "iot-stream-analysis-driver-100", "Domainname": "", "User": "0", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "KUBERNETES_SERVICE_PORT=443", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040", "KUBERNETES_PORT_443_TCP_PORT=443", "KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040", "KUBERNETES_SERVICE_HOST=161.161.0.1", "KUBERNETES_SERVICE_PORT_HTTPS=443", "KUBERNETES_PORT=tcp://161.161.0.1:443", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152", "KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443", "KUBERNETES_PORT_443_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin", "LANG=C.UTF-8", "JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk", "JAVA_VERSION=8u131", "JAVA_ALPINE_VERSION=8.131.11-r2", "SPARK_HOME=/opt/spark" ], "Cmd": [ "/etc/spark-init/spark-init.properties" ], "Healthcheck": { "Test": [ "NONE" ] }, "Image": "registry.dev161.com:5000/iot/spark-init@sha256:adca6ed85868d821e3f63e49b7d44765954bcf5e4131a4d419cd9f4dc2235401", "Volumes": null, "WorkingDir": "/opt/spark/work-dir", "Entrypoint": [ "/opt/entrypoint.sh", "/opt/spark/bin/spark-class", "org.apache.spark.deploy.rest.k8s.KubernetesSparkDependencyDownloadInitContainer" ], "OnBuild": null, "Labels": { "annotation.io.kubernetes.container.hash": "7757e3a7", "annotation.io.kubernetes.container.restartCount": "0", "annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log", "annotation.io.kubernetes.container.terminationMessagePolicy": "File", "annotation.io.kubernetes.pod.terminationGracePeriod": "30", "io.kubernetes.container.logpath": "/var/log/pods/64491f22-b575-11e8-910e-0050568848e8/spark-init_0.log", "io.kubernetes.container.name": "spark-init", "io.kubernetes.docker.type": "container", "io.kubernetes.pod.name": "iot-stream-analysis-driver-100", "io.kubernetes.pod.namespace": "spark-cluster", "io.kubernetes.pod.uid": "64491f22-b575-11e8-910e-0050568848e8", "io.kubernetes.sandbox.id": "d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": null, "SandboxKey": "", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": {} } }]driver:
[ { "Id": "2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c", "Created": "2018-09-11T03:47:46.771489986Z", "Path": "/opt/entrypoint.sh", "Args": [ "/bin/sh", "-c", "SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_DRIVER_JAVA_OPTS[@]}\" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY -Dspark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS" ], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 832405, "ExitCode": 0, "Error": "", "StartedAt": "2018-09-11T03:47:47.066061613Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:4ba1d64922f4ccfb682ca09d7d716252f41c55a650884b796598bafb03174805", "ResolvConfPath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/resolv.conf", "HostnamePath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/hostname", "HostsPath": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts", "LogPath": "/var/lib/docker/containers/2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c/2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c-json.log", "Name": "/k8s_spark-kubernetes-driver_iot-stream-analysis-driver-100_spark-cluster_64491f22-b575-11e8-910e-0050568848e8_0", "RestartCount": 0, "Driver": "overlay", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [ "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58:/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp:/var/run/secrets/kubernetes.io/serviceaccount:ro", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts", "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-kubernetes-driver/65c96924:/dev/termination-log" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": { "max-file": "5", "max-size": "50m" } }, "NetworkMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b", "PortBindings": null, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b", "Cgroup": "", "Links": null, "OomScoreAdj": 936, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": [ "seccomp=unconfined" ], "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 1024, "Memory": 1476395008, "NanoCpus": 0, "CgroupParent": "kubepods-burstable-pod64491f22_b575_11e8_910e_0050568848e8.slice", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": [], "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 2952790016, "MemorySwappiness": -1, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0 }, "GraphDriver": { "Name": "overlay", "Data": { "LowerDir": "/var/lib/docker/overlay/e7d727c6b28f718f78192d5bc76f24dffcf4603fba6e6173bded398b95d4c578/root", "MergedDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/merged", "UpperDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/upper", "WorkDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/work" } }, "Mounts": [ { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp", "Destination": "/var/run/secrets/kubernetes.io/serviceaccount", "Mode": "ro", "RW": false, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts", "Destination": "/etc/hosts", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-kubernetes-driver/65c96924", "Destination": "/dev/termination-log", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Destination": "/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume", "Destination": "/var/spark-data/spark-jars", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files", "Destination": "/var/spark-data/spark-files", "Mode": "", "RW": true, "Propagation": "" } ], "Config": { "Hostname": "iot-stream-analysis-driver-100", "Domainname": "", "User": "0", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "SPARK_DRIVER_BIND_ADDRESS=161.161.104.67", "SPARK_JAVA_OPT_9=-Dspark.kubernetes.initcontainer.docker.image=registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1", "SPARK_JAVA_OPT_10=-Dspark.kubernetes.driver.pod.name=iot-stream-analysis-driver-100", "SPARK_JAVA_OPT_13=-Dspark.kubernetes.docker.image.pullPolicy=Always", "SPARK_JAVA_OPT_14=-Dspark.kubernetes.driver.docker.image=registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1", "SPARK_JAVA_OPT_29=-Dspark.kubernetes.executor.docker.image=registry.dev161.com:5000/iot/spark-executor:v2.2.0-kubernetes-0.5.1", "SPARK_DRIVER_MEMORY=1024m", "SPARK_JAVA_OPT_1=-Dspark.kubernetes.svc.domain=svc.dev161", "SPARK_JAVA_OPT_5=-Dspark.local.dir=/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "SPARK_JAVA_OPT_11=-Dspark.app.name=spark-job-on-k8s-100", "SPARK_JAVA_OPT_19=-Dspark.kubernetes.executor.podNamePrefix=spark-job-on-k8s-100-1536637851433", "SPARK_JAVA_OPT_22=-Dspark.driver.port=7078", "SPARK_JAVA_OPT_25=-Dspark.executor.memory=1024m", "SPARK_MOUNTED_FILES_DIR=/var/spark-data/spark-files", "SPARK_JAVA_OPT_0=-Dspark.driver.host=spark-job-on-k8s-100-1536637851433-driver-svc.spark-cluster.svc.dev161", "SPARK_JAVA_OPT_3=-Dspark.submit.deployMode=cluster", "SPARK_JAVA_OPT_7=-Dspark.kubernetes.resourceStagingServer.uri=http://10.110.20.161:31000", "SPARK_JAVA_OPT_27=-Dspark.app.id=spark-1619ed3f66b04bf9a54a7230a1f551a4", "SPARK_JAVA_OPT_20=-Dspark.driver.memory=1024m", "SPARK_JAVA_OPT_21=-Dspark.kubernetes.submission.waitAppCompletion=true", "SPARK_JAVA_OPT_26=-Dspark.ui.enabled=false", "SPARK_MOUNTED_CLASSPATH=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar", "SPARK_JAVA_OPT_2=-Dspark.kubernetes.initcontainer.executor.configmapkey=download-submitted-files", "SPARK_JAVA_OPT_18=-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.name=spark-job-on-k8s-100-1536637851433-init-secret", "SPARK_JAVA_OPT_23=-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.mountDir=/mnt/secrets/spark-init", "SPARK_JAVA_OPT_28=-Dspark.executor.instances=1", "SPARK_JAVA_OPT_30=-Dspark.executor.cores=1", "SPARK_DRIVER_ARGS=--base64=true --rule=c2VsZWN0IHRpbWVTdGFtcCBBcyBrZXksIGNvbmNhdF93cygifCIsIHN0YXRlLnJlcG9ydGVkLnRlbXBlcmF0dXJlLCBjbGllbnRUb2tlbikgYXMgdmFsdWUgZnJvbSB0b3BpY3M= --sample={\"timeStamp\":1531381822,\"clientToken\":\"clientId_lamp\",\"state\":{\"reported\":{\"temperature\":23}}} --source-type=kafka --source={\"kafka.bootstrap.servers\":\"fnd-kafka-svc.default:9092\",\"subscribe\":\"sensor\"} --sink-type=console --verbose", "SPARK_JAVA_OPT_8=-Dspark.jars=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar", "SPARK_JAVA_OPT_12=-Dspark.driver.cores=1", "SPARK_JAVA_OPT_15=-Dspark.kubernetes.driver.nodeport=31202", "SPARK_JAVA_OPT_17=-Dspark.kubernetes.namespace=spark-cluster", "SPARK_JAVA_OPT_4=-Dspark.kubernetes.authenticate.driver.serviceAccountName=spark", "SPARK_JAVA_OPT_6=-Dspark.kubernetes.initcontainer.executor.configmapname=spark-job-on-k8s-100-1536637851433-init-config", "SPARK_JAVA_OPT_16=-Dspark.driver.blockManager.port=7079", "SPARK_DRIVER_CLASS=com.inspur.iot.RuleEngine", "SPARK_JAVA_OPT_24=-Dspark.master=k8s://http://10.110.20.161:8080", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "KUBERNETES_SERVICE_PORT_HTTPS=443", "KUBERNETES_PORT_443_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040", "KUBERNETES_PORT=tcp://161.161.0.1:443", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "KUBERNETES_SERVICE_PORT=443", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152", "KUBERNETES_SERVICE_HOST=161.161.0.1", "KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443", "KUBERNETES_PORT_443_TCP_PORT=443", "KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin", "LANG=C.UTF-8", "JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk", "JAVA_VERSION=8u131", "JAVA_ALPINE_VERSION=8.131.11-r2", "SPARK_HOME=/opt/spark" ], "Cmd": [ "/bin/sh", "-c", "SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_DRIVER_JAVA_OPTS[@]}\" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY -Dspark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS" ], "Healthcheck": { "Test": [ "NONE" ] }, "ArgsEscaped": true, "Image": "registry.dev161.com:5000/iot/spark-driver@sha256:6cf4ab9fd7ff1c21ddd1d66feb76107928c844cfaae912172bc618734942ff80", "Volumes": null, "WorkingDir": "/opt/spark/work-dir", "Entrypoint": [ "/opt/entrypoint.sh" ], "OnBuild": null, "Labels": { "annotation.io.kubernetes.container.hash": "57d51e36", "annotation.io.kubernetes.container.restartCount": "0", "annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log", "annotation.io.kubernetes.container.terminationMessagePolicy": "File", "annotation.io.kubernetes.pod.terminationGracePeriod": "30", "io.kubernetes.container.logpath": "/var/log/pods/64491f22-b575-11e8-910e-0050568848e8/spark-kubernetes-driver_0.log", "io.kubernetes.container.name": "spark-kubernetes-driver", "io.kubernetes.docker.type": "container", "io.kubernetes.pod.name": "iot-stream-analysis-driver-100", "io.kubernetes.pod.namespace": "spark-cluster", "io.kubernetes.pod.uid": "64491f22-b575-11e8-910e-0050568848e8", "io.kubernetes.sandbox.id": "d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": null, "SandboxKey": "", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": {} } }]executor:
[ { "Id": "37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322", "Created": "2018-09-11T03:47:55.861508477Z", "Path": "/opt/entrypoint.sh", "Args": [ "/bin/sh", "-c", "SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_EXECUTOR_JAVA_OPTS[@]}\" -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP" ], "State": { "Status": "running", "Running": true, "Paused": false, "Restarting": false, "OOMKilled": false, "Dead": false, "Pid": 11397, "ExitCode": 0, "Error": "", "StartedAt": "2018-09-11T03:47:55.982525484Z", "FinishedAt": "0001-01-01T00:00:00Z" }, "Image": "sha256:c45bb8d3fb02e8684a36f5cc02d00548ddeffe1102bd8e80e330ef715c470de9", "ResolvConfPath": "/var/lib/docker/containers/0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b/resolv.conf", "HostnamePath": "/var/lib/docker/containers/0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b/hostname", "HostsPath": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts", "LogPath": "/var/lib/docker/containers/37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322/37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322-json.log", "Name": "/k8s_executor_spark-job-on-k8s-100-1536637851433-exec-1_spark-cluster_74a94609-b575-11e8-910e-0050568848e8_0", "RestartCount": 0, "Driver": "overlay", "MountLabel": "", "ProcessLabel": "", "AppArmorProfile": "", "ExecIDs": null, "HostConfig": { "Binds": [ "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58:/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/default-token-jxg4k:/var/run/secrets/kubernetes.io/serviceaccount:ro", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts", "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/containers/executor/664b5635:/dev/termination-log" ], "ContainerIDFile": "", "LogConfig": { "Type": "json-file", "Config": { "max-file": "5", "max-size": "50m" } }, "NetworkMode": "container:0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b", "PortBindings": null, "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "AutoRemove": false, "VolumeDriver": "", "VolumesFrom": null, "CapAdd": null, "CapDrop": null, "Dns": null, "DnsOptions": null, "DnsSearch": null, "ExtraHosts": null, "GroupAdd": null, "IpcMode": "container:0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b", "Cgroup": "", "Links": null, "OomScoreAdj": 936, "PidMode": "", "Privileged": false, "PublishAllPorts": false, "ReadonlyRootfs": false, "SecurityOpt": [ "seccomp=unconfined" ], "UTSMode": "", "UsernsMode": "", "ShmSize": 67108864, "Runtime": "runc", "ConsoleSize": [ 0, 0 ], "Isolation": "", "CpuShares": 1024, "Memory": 1476395008, "NanoCpus": 0, "CgroupParent": "kubepods-burstable-pod74a94609_b575_11e8_910e_0050568848e8.slice", "BlkioWeight": 0, "BlkioWeightDevice": null, "BlkioDeviceReadBps": null, "BlkioDeviceWriteBps": null, "BlkioDeviceReadIOps": null, "BlkioDeviceWriteIOps": null, "CpuPeriod": 0, "CpuQuota": 0, "CpuRealtimePeriod": 0, "CpuRealtimeRuntime": 0, "CpusetCpus": "", "CpusetMems": "", "Devices": [], "DiskQuota": 0, "KernelMemory": 0, "MemoryReservation": 0, "MemorySwap": 2952790016, "MemorySwappiness": -1, "OomKillDisable": false, "PidsLimit": 0, "Ulimits": null, "CpuCount": 0, "CpuPercent": 0, "IOMaximumIOps": 0, "IOMaximumBandwidth": 0 }, "GraphDriver": { "Name": "overlay", "Data": { "LowerDir": "/var/lib/docker/overlay/9e0164c7e2d049bbf3260f6a332b2995368d8af4d6f8357f8c70c18f8d4c7864/root", "MergedDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/merged", "UpperDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/upper", "WorkDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/work" } }, "Mounts": [ { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume", "Destination": "/var/spark-data/spark-jars", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files", "Destination": "/var/spark-data/spark-files", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/default-token-jxg4k", "Destination": "/var/run/secrets/kubernetes.io/serviceaccount", "Mode": "ro", "RW": false, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts", "Destination": "/etc/hosts", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/containers/executor/664b5635", "Destination": "/dev/termination-log", "Mode": "", "RW": true, "Propagation": "" }, { "Type": "bind", "Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Destination": "/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58", "Mode": "", "RW": true, "Propagation": "" } ], "Config": { "Hostname": "spark-job-on-k8s-100-1536637851433-exec-1", "Domainname": "", "User": "0", "AttachStdin": false, "AttachStdout": false, "AttachStderr": false, "Tty": false, "OpenStdin": false, "StdinOnce": false, "Env": [ "SPARK_EXECUTOR_ID=1", "SPARK_EXECUTOR_POD_IP=161.161.76.219", "SPARK_MOUNTED_FILES_DIR=/var/spark-data/spark-files", "SPARK_EXECUTOR_PORT=10000", "SPARK_DRIVER_URL=spark://CoarseGrainedScheduler@spark-job-on-k8s-100-1536637851433-driver-svc.spark-cluster.svc.dev161:7078", "SPARK_EXECUTOR_CORES=1", "SPARK_EXECUTOR_MEMORY=1024m", "SPARK_APPLICATION_ID=spark-1619ed3f66b04bf9a54a7230a1f551a4", "SPARK_MOUNTED_CLASSPATH=/var/spark-data/spark-jars/*", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23", "KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040", "KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443", "KUBERNETES_PORT_443_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040", "KUBERNETES_SERVICE_HOST=161.161.0.1", "KUBERNETES_SERVICE_PORT_HTTPS=443", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152", "COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040", "KUBERNETES_SERVICE_PORT=443", "KUBERNETES_PORT=tcp://161.161.0.1:443", "KUBERNETES_PORT_443_TCP_PORT=443", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040", "COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin", "LANG=C.UTF-8", "JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk", "JAVA_VERSION=8u131", "JAVA_ALPINE_VERSION=8.131.11-r2", "SPARK_HOME=/opt/spark" ], "Cmd": [ "/bin/sh", "-c", "SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_EXECUTOR_JAVA_OPTS[@]}\" -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP" ], "Healthcheck": { "Test": [ "NONE" ] }, "ArgsEscaped": true, "Image": "registry.dev161.com:5000/iot/spark-executor@sha256:c8650704517e254751fcef052eff13e3e4e40cff4b02039aceb9ce050f77d9a3", "Volumes": null, "WorkingDir": "/opt/spark/work-dir", "Entrypoint": [ "/opt/entrypoint.sh" ], "OnBuild": null, "Labels": { "annotation.io.kubernetes.container.hash": "3136f4c3", "annotation.io.kubernetes.container.ports": "[{\"name\":\"executor\",\"containerPort\":10000,\"protocol\":\"TCP\"},{\"name\":\"blockmanager\",\"containerPort\":7079,\"protocol\":\"TCP\"}]", "annotation.io.kubernetes.container.restartCount": "0", "annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log", "annotation.io.kubernetes.container.terminationMessagePolicy": "File", "annotation.io.kubernetes.pod.terminationGracePeriod": "30", "io.kubernetes.container.logpath": "/var/log/pods/74a94609-b575-11e8-910e-0050568848e8/executor_0.log", "io.kubernetes.container.name": "executor", "io.kubernetes.docker.type": "container", "io.kubernetes.pod.name": "spark-job-on-k8s-100-1536637851433-exec-1", "io.kubernetes.pod.namespace": "spark-cluster", "io.kubernetes.pod.uid": "74a94609-b575-11e8-910e-0050568848e8", "io.kubernetes.sandbox.id": "0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b" } }, "NetworkSettings": { "Bridge": "", "SandboxID": "", "HairpinMode": false, "LinkLocalIPv6Address": "", "LinkLocalIPv6PrefixLen": 0, "Ports": null, "SandboxKey": "", "SecondaryIPAddresses": null, "SecondaryIPv6Addresses": null, "EndpointID": "", "Gateway": "", "GlobalIPv6Address": "", "GlobalIPv6PrefixLen": 0, "IPAddress": "", "IPPrefixLen": 0, "IPv6Gateway": "", "MacAddress": "", "Networks": {} } }]resource-staging-server:
apiVersion: v1kind: Podmetadata: creationTimestamp: 2018-09-11T10:39:29Z generateName: spark-resource-staging-server-544f9dfdc7- labels: pod-template-hash: "1009589873" resource-staging-server-instance: default name: spark-resource-staging-server-544f9dfdc7-6z9qm namespace: iop ownerReferences: - apiVersion: extensions/v1beta1 blockOwnerDeletion: true controller: true kind: ReplicaSet name: spark-resource-staging-server-544f9dfdc7 uid: f64cb936-b5ae-11e8-a973-6c92bf88f88e resourceVersion: "5271010" selfLink: /api/v1/namespaces/iop/pods/spark-resource-staging-server-544f9dfdc7-6z9qm uid: f64ec3f8-b5ae-11e8-a973-6c92bf88f88espec: containers: - args: - /etc/spark-resource-staging-server/resource-staging-server.properties image: registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server:v2.2.0-kubernetes-0.4.0 imagePullPolicy: IfNotPresent name: spark-resource-staging-server resources: limits: cpu: "1" memory: 2Gi requests: cpu: 100m memory: 256Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /etc/spark-resource-staging-server name: resource-staging-server-properties - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: default-token-ztnhf readOnly: true dnsPolicy: ClusterFirst imagePullSecrets: - name: service-registry nodeName: slave1 restartPolicy: Always schedulerName: default-scheduler securityContext: {} serviceAccount: default serviceAccountName: default terminationGracePeriodSeconds: 30 volumes: - configMap: defaultMode: 420 name: spark-resource-staging-server-config name: resource-staging-server-properties - name: default-token-ztnhf secret: defaultMode: 420 secretName: default-token-ztnhfstatus: conditions: - lastProbeTime: null lastTransitionTime: 2018-09-11T10:39:29Z status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: 2018-09-11T10:39:42Z status: "True" type: Ready - lastProbeTime: null lastTransitionTime: 2018-09-11T10:39:29Z status: "True" type: PodScheduled containerStatuses: - containerID: docker://433aec71282a7371c091c69f7f921a5612f2ac98ec26c758629836c4855ebb1f image: registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server:v2.2.0-kubernetes-0.4.0 imageID: docker-pullable://registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server@sha256:9480a49fb7c1ff6d6da85dcdc034a2e0a0006fb3d3a424671bf174c055b538d3 lastState: {} name: spark-resource-staging-server ready: true restartCount: 0 state: running: startedAt: 2018-09-11T10:39:42Z hostIP: 10.100.0.17 phase: Running podIP: 214.213.140.247 qosClass: Burstable startTime: 2018-09-11T10:39:29Z
源代码
数据库的安全要保护哪些东西
数据库安全各自的含义是什么
生产安全数据库录入
数据库的安全性及管理
数据库安全策略包含哪些
海淀数据库安全审计系统
建立农村房屋安全信息数据库
易用的数据库客户端支持安全管理
连接数据库失败ssl安全错误
数据库的锁怎样保障安全
临猗奖舅网络技术有限公司
将自己的电脑配置为公网服务器
软件开发和电子商务有关联吗
家庭nas服务器有用吗
江苏学生网络技术咨询指导
数据库服务器断电
全国信托数据库
cmd切换数据库
关于网络安全复试的问题
梦幻西游端游数据库
软件开发竞价
软件开发和嵌入式
村镇银行网络安全风险评估
常州营销网络技术收费
虚拟空间传输服务器
铜川软件开发电话
轩雨阁网络技术服务
小学生国家网络安全班会课教案
php 数据库类
lol云顶之弈测试服务器
流行病学有成熟的数据库吗
druid 数据库不可用
网络安全法行为名称
经济学常用围观数据库
网络技术的发展轨迹思维导图
巴中软件开发方案
南开网考网络技术与应用
企业级软件开发招聘
数据库不显示内容是
5条关于网络安全的
