I write code invoke the html5 parser twice with the same of html.The the following output show the tokenizer.js still have a bug:
DEBUG: 'tokenizer.content_model=' 0
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'markup_declaration_open_state'
DEBUG: 'tokenizer.state=' 'doctype_state'
DEBUG: 'tokenizer.state=' 'before_doctype_name_state'
DEBUG: 'tokenizer.state=' 'doctype_name_state'
DEBUG: 'tokenizer.token' { type: 'Doctype',
name: 'html',
publicId: null,
systemId: null,
correct: true }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'markup_declaration_open_state'
DEBUG: 'tokenizer.state=' 'comment_start_state'
DEBUG: 'tokenizer.state=' 'comment_state'
DEBUG: 'tokenizer.state=' 'comment_end_dash_state'
DEBUG: 'tokenizer.state=' 'comment_end_state'
DEBUG: 'tokenizer.token' { type: 'Comment', data: 'NewPage' }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'StartTag', name: 'html', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: ' ' }
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: ' ' }
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'markup_declaration_open_state'
DEBUG: 'tokenizer.state=' 'comment_start_state'
DEBUG: 'tokenizer.state=' 'comment_state'
DEBUG: 'tokenizer.state=' 'comment_end_dash_state'
DEBUG: 'tokenizer.state=' 'comment_end_state'
DEBUG: 'tokenizer.token' { type: 'Comment',
data: '[if ie 6]>\n\n<![endif]' }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'StartTag', name: 'body', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'StartTag', name: 'script', data: [] }
DEBUG: 'tokenizer.content_model=' 3
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'close_tag_open_state'
DEBUG: 'tokenizer.content_model=' 0
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'Characters',
data: 'var a = 1;\nvar b = 2;\n' }
DEBUG: 'tokenizer.token' { type: 'EndTag', name: 'script', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.state=' 'before_attribute_name_state'
DEBUG: 'tokenizer.state=' 'attribute_name_state'
DEBUG: 'tokenizer.state=' 'before_attribute_value_state'
DEBUG: 'tokenizer.state=' 'attribute_value_double_quoted_state'
DEBUG: 'tokenizer.state=' 'after_attribute_value_state'
DEBUG: 'tokenizer.state=' 'before_attribute_name_state'
DEBUG: 'tokenizer.state=' 'attribute_name_state'
DEBUG: 'tokenizer.state=' 'before_attribute_value_state'
DEBUG: 'tokenizer.state=' 'attribute_value_double_quoted_state'
DEBUG: 'tokenizer.state=' 'after_attribute_value_state'
DEBUG: 'tokenizer.token' { type: 'StartTag',
name: 'p',
data:
[ { nodeName: 'class', nodeValue: 'po' },
{ nodeName: 'id', nodeValue: 'p' } ] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: ' ' }
DEBUG: 'tokenizer.token' { type: 'Characters', data: 'element ' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'close_tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'EndTag', name: 'p', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.token' { type: 'EOF', data: 'End of File' }
DEBUG: 'tokenizer.content_model=' 0
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'markup_declaration_open_state'
DEBUG: 'tokenizer.state=' 'doctype_state'
DEBUG: 'tokenizer.state=' 'before_doctype_name_state'
DEBUG: 'tokenizer.state=' 'doctype_name_state'
DEBUG: 'tokenizer.token' { type: 'Doctype',
name: 'html',
publicId: null,
systemId: null,
correct: true }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'markup_declaration_open_state'
DEBUG: 'tokenizer.state=' 'comment_start_state'
DEBUG: 'tokenizer.state=' 'comment_state'
DEBUG: 'tokenizer.state=' 'comment_end_dash_state'
DEBUG: 'tokenizer.state=' 'comment_end_state'
DEBUG: 'tokenizer.token' { type: 'Comment', data: 'NewPage' }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'StartTag', name: 'html', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: ' ' }
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: ' ' }
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'markup_declaration_open_state'
DEBUG: 'tokenizer.state=' 'comment_start_state'
DEBUG: 'tokenizer.state=' 'comment_state'
DEBUG: 'tokenizer.state=' 'comment_end_dash_state'
DEBUG: 'tokenizer.state=' 'comment_end_state'
DEBUG: 'tokenizer.token' { type: 'Comment',
data: '[if ie 6]>\n\n<![endif]' }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'StartTag', name: 'body', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'StartTag', name: 'script', data: [] }
DEBUG: 'tokenizer.content_model=' 3
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.token' { type: 'Characters',
data: 'var a = 1;\nvar b = 2;\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'close_tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'Characters', data: undefined }
DEBUG: 'tokenizer.token' { type: 'EndTag', name: 'script', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.state=' 'before_attribute_name_state'
DEBUG: 'tokenizer.state=' 'attribute_name_state'
DEBUG: 'tokenizer.state=' 'before_attribute_value_state'
DEBUG: 'tokenizer.state=' 'attribute_value_double_quoted_state'
DEBUG: 'tokenizer.state=' 'after_attribute_value_state'
DEBUG: 'tokenizer.state=' 'before_attribute_name_state'
DEBUG: 'tokenizer.state=' 'attribute_name_state'
DEBUG: 'tokenizer.state=' 'before_attribute_value_state'
DEBUG: 'tokenizer.state=' 'attribute_value_double_quoted_state'
DEBUG: 'tokenizer.state=' 'after_attribute_value_state'
DEBUG: 'tokenizer.token' { type: 'StartTag',
name: 'p',
data:
[ { nodeName: 'class', nodeValue: 'po' },
{ nodeName: 'id', nodeValue: 'p' } ] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: ' ' }
DEBUG: 'tokenizer.token' { type: 'Characters', data: 'element ' }
DEBUG: 'tokenizer.state=' 'tag_open_state'
DEBUG: 'tokenizer.state=' 'close_tag_open_state'
DEBUG: 'tokenizer.state=' 'tag_name_state'
DEBUG: 'tokenizer.token' { type: 'EndTag', name: 'p', data: [] }
DEBUG: 'tokenizer.state=' 'data_state'
DEBUG: 'tokenizer.token' { type: 'SpaceCharacters', data: '\n' }
DEBUG: 'tokenizer.token' { type: 'EOF', data: 'End of File' }
The code runs different way in Script element with the characters token.