conversant / spark-profiler Goto Github PK

Sample event of spark 2.1.0 :

{"Event":"SparkListenerTaskEnd","Stage ID":0,"Stage Attempt ID":0,"Task Type":"ShuffleMapTask","Task End Reason":{"Reason":"Success"},"Task Info":{"Task ID":0,"Index":0,"Attempt":0,"Launch Time":1511259408672,"Executor ID":"1","Host":"sandbox.hortonworks.com","Locality":"PROCESS_LOCAL","Speculative":false,"Getting Result Time":0,"Finish Time":1511259410923,"Failed":false,"Killed":false,"Accumulables":[{"ID":13,"Name":"internal.metrics.executorDeserializeTime","Update":1400,"Value":1400,"Internal":true,"Count Failed Values":true},{"ID":14,"Name":"internal.metrics.executorDeserializeCpuTime","Update":992315389,"Value":992315389,"Internal":true,"Count Failed Values":true},{"ID":15,"Name":"internal.metrics.executorRunTime","Update":677,"Value":677,"Internal":true,"Count Failed Values":true},{"ID":16,"Name":"internal.metrics.executorCpuTime","Update":606198111,"Value":606198111,"Internal":true,"Count Failed Values":true},{"ID":17,"Name":"internal.metrics.resultSize","Update":2705,"Value":2705,"Internal":true,"Count Failed Values":true},{"ID":18,"Name":"internal.metrics.jvmGCTime","Update":65,"Value":65,"Internal":true,"Count Failed Values":true},{"ID":19,"Name":"internal.metrics.resultSerializationTime","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":23,"Name":"internal.metrics.updatedBlockStatuses","Update":[{"Block ID":"broadcast_0_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":4659,"Disk Size":0}},{"Block ID":"broadcast_0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":8952,"Disk Size":0}}],"Value":[{"Block ID":"broadcast_0_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":4659,"Disk Size":0}},{"Block ID":"broadcast_0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":8952,"Disk Size":0}}],"Internal":true,"Count Failed Values":true},{"ID":30,"Name":"internal.metrics.shuffle.write.bytesWritten","Update":59,"Value":59,"Internal":true,"Count Failed Values":true},{"ID":31,"Name":"internal.metrics.shuffle.write.recordsWritten","Update":1,"Value":1,"Internal":true,"Count Failed Values":true},{"ID":32,"Name":"internal.metrics.shuffle.write.writeTime","Update":7044541,"Value":7044541,"Internal":true,"Count Failed Values":true},{"ID":12,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":7,"Name":"duration total (min, med, max)","Update":"37","Value":"36","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":8,"Name":"number of output rows","Update":"1","Value":"1","Internal":true,"Count Failed Values":true,"Metadata":"sql"},{"ID":1,"Name":"data size total (min, med, max)","Update":"15","Value":"14","Internal":true,"Count Failed Values":true,"Metadata":"sql"}]},"Task Metrics":{"Executor Deserialize Time":1400,"Executor Deserialize CPU Time":992315389,"Executor Run Time":677,"Executor CPU Time":606198111,"Result Size":2705,"JVM GC Time":65,"Result Serialization Time":1,"Memory Bytes Spilled":0,"Disk Bytes Spilled":0,"Shuffle Read Metrics":{"Remote Blocks Fetched":0,"Local Blocks Fetched":0,"Fetch Wait Time":0,"Remote Bytes Read":0,"Local Bytes Read":0,"Total Records Read":0},"Shuffle Write Metrics":{"Shuffle Bytes Written":59,"Shuffle Write Time":7044541,"Shuffle Records Written":1},"Input Metrics":{"Bytes Read":0,"Records Read":0},"Output Metrics":{"Bytes Written":0,"Records Written":0},"Updated Blocks":[{"Block ID":"broadcast_0_piece0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":false,"Replication":1},"Memory Size":4659,"Disk Size":0}},{"Block ID":"broadcast_0","Status":{"Storage Level":{"Use Disk":false,"Use Memory":true,"Deserialized":true,"Replication":1},"Memory Size":8952,"Disk Size":0}}]}}

The event has varying data type for update field thats why getting error:

Caused by: org.json4s.package$MappingException: Expected collection but got JInt(16959) for root JInt(16959) and mapping Seq[SparkAccumulableUpdate]
No usable value for Task Info
No usable value for Accumulables
No usable value for Update
Expected collection but got JInt(16959) for root JInt(16959) and mapping Seq[SparkAccumulableUpdate]

Can you please check it against spark 2.1.0, any help would be appreciated.