tautologistics / node-htmlparser Goto Github PK
View Code? Open in Web Editor NEWForgiving HTML/XML/RSS Parser in JS for *both* Node and Browsers
License: MIT License
Forgiving HTML/XML/RSS Parser in JS for *both* Node and Browsers
License: MIT License
var htmlparser = require('htmlparser'),
util = require('util'),
handler = new htmlparser.DefaultHandler(function(err, dom){}),
parser = new htmlparser.Parser(handler),
rawHtml = '<script type="text/template"><h1>Heading1</h1></script>';
parser.parseComplete(rawHtml);
console.log(util.inspect(handler.dom, false, null));
This piece of code discards "<" of <h1> and outputs:
[ { raw: 'script type="text/template"',
data: 'script type="text/template"',
type: 'script',
name: 'script',
attribs: { type: 'text/template' },
children:
[ { raw: 'h1>Heading1</h1>', // discard <
data: 'h1>Heading1</h1>',
type: 'text' } ] } ]
in the older version of html parser, it was output as a directive, however in the current trunk, it doesn't do anything.
example:
npm install htmlparser will install v1.7.3 but v2.0.0,how can i get the newest version?
var jsdom = require('jsdom').jsdom;
var doc = jsdom('<html><body><input value="<"></body></html>');
issue discovered originally in JSDOM which uses html parser:
How to detect a form which is created in javascript? I am unable o detect it.
For eg:
var frm = document.createElement("form")
Based on the example in http://www.whatwg.org/specs/web-apps/current-work/#attr-iframe-srcdoc :
var rawHtml = '<iframe srcdoc="<p>Yeah, you can see it <a href="/gallery?mode=cover&amp;page=1">in my gallery</a>."></iframe>',
htmlparser = require('./lib/htmlparser'),
handler = new htmlparser.DefaultHandler(),
parser = new htmlparser.Parser(handler);
parser.parseComplete(rawHtml);
console.warn(require('util').inspect(handler.dom, false, null));
Output:
[ { raw: 'iframe srcdoc="',
data: 'iframe srcdoc="',
type: 'tag',
name: 'iframe',
attribs: { srcdoc: 'srcdoc' },
children:
[ { raw: 'p',
data: 'p',
type: 'tag',
name: 'p',
children:
[ { raw: 'Yeah, you can see it ',
data: 'Yeah, you can see it ',
type: 'text' },
{ raw: 'a href="/gallery?mode=cover&amp;page=1"',
data: 'a href="/gallery?mode=cover&amp;page=1"',
type: 'tag',
name: 'a',
attribs: { href: '"/gallery?mode=cover&amp;page=1"' },
children: [ { raw: 'in my gallery', data: 'in my gallery', type: 'text' } ] },
{ raw: '."', data: '."', type: 'text' } ] } ] } ]
Expected output:
[ { raw: 'iframe srcdoc="<p>Yeah, you can see it <a href="/gallery?mode=cover&amp;page=1">in my gallery</a>."',
data: 'iframe srcdoc="<p>Yeah, you can see it <a href="/gallery?mode=cover&amp;page=1">in my gallery</a>."',
type: 'tag',
name: 'iframe',
attribs: { srcdoc: '<p>Yeah, you can see it <a href="/gallery?mode=cover&amp;page=1">in my gallery</a>.' } } ]
It works if I entitify the less thans and greater thans.
or change lib/node-htmlparser.js to lib/htmlparser.js so I can localize / expose via require.paths using require('htmlparser')
I install from npm, version is v1.7.6
<ul><li><ul><li>1<li>2</ul><li>3</ul>
test faild.
expected:
[
{
"raw": "ul",
"data": "ul",
"type": "tag",
"name": "ul",
"children": [
{
"raw": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"raw": "ul",
"data": "ul",
"type": "tag",
"name": "ul",
"children": [
{
"row": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"row": "1",
"data": "1",
"type": "text"
}
]
},
{
"row": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"row": "2",
"data": "2",
"type": "text"
}
]
}
]
}
]
},
{
"raw": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"raw": "3",
"data": "3",
"type": "text"
}
]
}
]
}
]
Complete
[
{
"raw": "ul",
"data": "ul",
"type": "tag",
"name": "ul",
"children": [
{
"raw": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"raw": "ul",
"data": "ul",
"type": "tag",
"name": "ul",
"children": [
{
"raw": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"raw": "1",
"data": "1",
"type": "text"
},
{
"raw": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"raw": "2",
"data": "2",
"type": "text"
}
]
}
]
}
]
},
{
"raw": "li",
"data": "li",
"type": "tag",
"name": "li",
"children": [
{
"raw": "3",
"data": "3",
"type": "text"
}
]
}
]
}
]
}
]
i have html that looks like the following:
<span title="first line<br>second line"></span>
it would be great if the 'br' is actually treated as part of the attribute value of 'title' instead of being treated as a new tag element
Hi there,
If you look at a live journal entry like this one: http://cananian.livejournal.com/60624.html
You can see the
When doing: description = htmlparser.DomUtils.getElements( { tag_name: "meta", name: "description" }, dom);
Instead of having a result like this:
[ { raw: 'meta name="description" value="the whole post in there"/', data: 'meta name="description" value="the whole post in there"/', type: 'tag', name: 'meta', attribs: { name: 'description', value: 'the whole post in there' } } ]
I have this:
[ { raw: 'meta name="description" value="the whole post in there"/', data: 'meta name="description" value="the whole post in there"/', type: 'tag', name: 'meta', attribs: { name: 'description', value: 'the whole post in there' , the: 'the', whole: 'whole', post: 'post', in: 'in', there: 'there' } } ]
Hope it helps!
var rawHtml = '<script>document.write("<!--hello-->");</script>',
htmlparser = require('./lib/htmlparser'),
handler = new htmlparser.DefaultHandler(),
parser = new htmlparser.Parser(handler);
parser.parseComplete(rawHtml);
console.warn(require('util').inspect(handler.dom, false, null));
Output:
[ { raw: 'script',
data: 'script',
type: 'script',
name: 'script',
children:
[ { raw: 'document.write("',
data: 'document.write("',
type: 'text' },
{ raw: 'hello', data: 'hello', type: 'comment' },
{ raw: '");', data: '");', type: 'text' } ] } ]
Expected output:
[ { raw: 'script',
data: 'script',
type: 'script',
name: 'script',
children:
[ { raw: 'document.write("<!--hello-->");',
data: 'document.write("<!--hello-->");',
type: 'text' } ] } ]
IE8 (at least) will take the following HTML:
<!DOCTYPE html>
<html>
<head></head>
<body></body>
</html>
And convert it to:
<!DOCTYPE HTML>
<HTML>
<HEAD></HEAD>
<BODY></BODY>
</HTML>
The neat part: node-htmlparser handles this just fine!
The bad: libraries like soupselect (https://github.com/harryf/node-soupselect) and the DomUtils included with node-htmlparser will fail when trying to select 'body'. The DomUtils will select 'BODY' properly, but it's a pain to have to try to select BOTH 'body' and 'BODY'.
Should this be handled on the parser-side of things or the selector side of things? I'm not sure. Part of me thinks that the parser should normalize the HTML to an extent, such as make all the tags lowercase. At the same time, a selector engine could do this normalization when searching.
Thoughts?
example:
<div by-zero name="something">
If a tag like this appears anywhere in the html, the parsing stops shortly after a tag like this. I saw this tag in the reddit source code where my code bombed.
http://blog.kickstarter.com/post/5770516169/new-projects-are-rolling-dice
I can't tell what's causing it, but the html gets parsed into approximately 16 root elements, instead of 2 (doctype, html). It could be CDATA, or the combination of crazy tumblr injected content near the bottom.
node has a streams API. why do you have custom stream api
I need to get text contents inside an element and before reinventing the wheel I must be sure if anything similiar exists.
Would it be useful if I create a few utility methods that can be merged with DomUtils?
Can you share an example of a working script using Streaming To Parser in Node?
Using your README text generates an error.
fs.createReadStream('./path_to_file.html').pipe(parser);
I get this error in console:
_stream_readable.js:476
dest.on('unpipe', onunpipe);
^
TypeError: Object #<Parser> has no method 'on'
at ReadStream.Readable.pipe (_stream_readable.js:476:8)
at Object.<anonymous> (C:\dev\autorelease\script.js:54:8)
at Module._compile (module.js:456:26)
at Object.Module._extensions..js (module.js:474:10)
at Module.load (module.js:356:32)
at Function.Module._load (module.js:312:12)
at Function.Module.runMain (module.js:497:10)
at startup (node.js:119:16)
at node.js:902:3
Here is my simple script:
// get a file stream reader
var reader = fs.createReadStream(process.argv[2]);
// get a file stream writer pointing to the json file to write to
var writer = fs.createWriteStream(input_json);
var htmlparser = require("htmlparser");
//var rawHtml = "Xyz <script language= javascript>var foo = '<<bar>>';< / script><!--<!-- Waah! -- -->";
//var sys = require("sys");
var handler = new htmlparser.DefaultHandler(function (error, dom) {
if (error)
logger.log('error', 'handler error in parser', {error: error});
else
logger.log('info', '', {dom: JSON.stringify(dom)});
});
var parser = new htmlparser.Parser(handler);
//parser.parseComplete(reader);
//sys.puts(sys.inspect(handler.dom, false, null));
// pipe everything to do the conversion
reader.pipe(parser).pipe(writer);
The wild internet sometimes contains weird stuff that makes this parser behave funny.
A tag such as this:
<a href="#" onclick="moveAddCommentBelow("div-comment-579747", 579747, true); return false;" />
Has attributes parsed like so:
{ href: '#'
, onclick: 'moveAddCommentBelow('
, 'div-comment-579747': 'div-comment-579747'
, ',': ','
, '579747,': '579747,'
, 'true);': 'true);'
, return: 'return'
, 'false;': 'false;'
}
I don't know whether I'm missing something or not, please tell me if I do and excuse me if this is too obvious but there is no DefaultHandler
method of htmlparser
object in master branch (version 2.0.0).
I've tried using this library in my browser but when I inspect htmlparser
(Tautologistics.NodeHtmlParser
) object, it doesn't have such a method. However it works like a charm in 1.x version!
is there something missing from this branch? or I'm missing something?
Thanks in advance.
for me. however when I change it to tagStack['-1']()
its fine, haha.. no clue what you are doing here, but yeah here is my trace. Which is used with jsdom for parsing even a simple string like <html><body><p>foo</p></body></html>
.
register.test.js test GET /signup: TypeError: Property 'last' of object [object Object] is not a function
at DefaultHandler.DefaultHandler$handleElement [as handleElement] (/Users/tj/Projects/LearnBoost/tests/functional/support/htmlparser/lib/htmlparser.js:694:26)
at DefaultHandler.DefaultHandler$writeTag [as writeTag] (/Users/tj/Projects/LearnBoost/tests/functional/support/htmlparser/lib/htmlparser.js:612:8)
at Parser.Parser$writeHandler [as writeHandler] (/Users/tj/Projects/LearnBoost/tests/functional/support/htmlparser/lib/htmlparser.js:443:20)
at Parser.Parser$parseTags [as parseTags] (/Users/tj/Projects/LearnBoost/tests/functional/support/htmlparser/lib/htmlparser.js:383:8)
at Parser.Parser$parseChunk [as parseChunk] (/Users/tj/Projects/LearnBoost/tests/functional/support/htmlparser/lib/htmlparser.js:95:8)
at Parser.Parser$parseComplete [as parseComplete] (/Users/tj/Projects/LearnBoost/tests/functional/support/htmlparser/lib/htmlparser.js:86:8)
at Object.ParseHtml (/Users/tj/Projects/LearnBoost/tests/functional/support/jsdom/lib/jsdom/browser/htmltodom.js:62:16)
at HtmlToDom.appendHtmlToElement (/Users/tj/Projects/LearnBoost/tests/functional/support/jsdom/lib/jsdom/browser/htmltodom.js:73:27)
at Object.innerHTML (/Users/tj/Projects/LearnBoost/tests/functional/support/jsdom/lib/jsdom/browser/index.js:295:27)
at Object.write (/Users/tj/Projects/LearnBoost/tests/functional/support/jsdom/lib/jsdom/browser/index.js:202:22)
at Object.jsdom (/Users/tj/Projects/LearnBoost/tests/functional/support/jsdom/lib/jsdom.js:30:9)
at /Users/tj/Projects/LearnBoost/tests/functional/register.test.js:26:23
at next (/Users/tj/Projects/LearnBoost/tests/integration/support/expresso/bin/expresso:769:25)
at runSuite (/Users/tj/Projects/LearnBoost/tests/integration/support/expresso/bin/expresso:787:6)
at check (/Users/tj/Projects/LearnBoost/tests/integration/support/expresso/bin/expresso:648:16)
at runFile (/Users/tj/Projects/LearnBoost/tests/integration/support/expresso/bin/expresso:652:10)
at Array.forEach (native)
at runFiles (/Users/tj/Projects/LearnBoost/tests/integration/support/expresso/bin/expresso:629:13)
at run (/Users/tj/Projects/LearnBoost/tests/integration/support/expresso/bin/expresso:598:5)
at Object.<anonymous> (/Users/tj/Projects/LearnBoost/tests/integration/support/expresso/bin/expresso:851:13)
at Module._compile (node.js:462:23)
at Module._loadScriptSync (node.js:469:10)
at Module.loadSync (node.js:338:12)
at Object.runMain (node.js:522:24)
at Array.0 (node.js:756:12)
at EventEmitter._tickCallback (node.js:55:22)
at node.js:773:9
I'm trying to integrate this into jsDOM to fix issues with parsing '<' or '>' in attributes.
After I successfully do that, I'll update the directions in the readme based on my experience.
EDIT: Also, if it's ready, can you publish to npm?
Hi I copied and pasted your example usage code for browsers and I get this error: TypeError: Tautologistics.NodeHtmlParser.DefaultHandler is not a constructor.
Can you verify that the test file works in v.1.2 ?
Any ideas why/where this is showing up?
I'm seeing a folder called "testdata" with dozens of links to torrents. I don't see it in source here on git for v 1.7.6, but npm is serving it up and my IDE is listing external dependencies for "show_ads.js", etc. We're also discussing this at https://groups.google.com/forum/#!topic/nodejs/0ZOIIHMp2_o
node-htmlparser
parses these two codes in the same way!
<span class="copyright link">Copyright content</span>
<span class="copyright link">Copyright content</spane>
Example: <memo if="i>1">...</memo>
Node-htmlparser doesn't decode character references, e.g. & or å .
Example: "
H1
#text( A&W )
The correct parse tree is:
H1
#text( A&W )
It seems like there are a number of high value pull requests and this lib hasn't been touched in a year.
If a text on the HTML has a "<", the text is not parsed after that.
Example: <title>We <3cupcakes</title>
The "<3cupcakes" is interpreted like being a tag when common browsers parse it like text.
I might be doing the readout wrong, but this is the second time I've picked this up. It seems that <source>
isn't identified as a void tag, so they become children of one another when listed inside a <video>
:
var htmlparser = require('htmlparser');
var htmlContent = "<html><head></head><body><video><source src=\"foo.ogv\"><source src=\"lol.smaz\"></video><div></div></body></html>";
var handler = new htmlparser.DefaultHandler(function (error, dom) {
function parse(dom, spacing){
console.log(spacing, dom.name);
if(dom.children){
for(var i=0; i<dom.children.length; ++i){
parse(dom.children[i], spacing + ' ');
}
}
}
parse(dom[0], '');
});
new htmlparser.Parser(handler).parseComplete(htmlContent);
Hey there,
I tried installing htmlparser with npm, and it had some issues:
Thanks!
Looks like you were testing with libxmljs.node in your local folder before running npm package
? I noticed this because I wrote a script to scan for C++ node modules and it bumped into this libxmljs.node, which seemed odd because htmlparser does not list it as a dependency in package.json. Just thought you might want to know.
As a suggestion to the DomUtils "submodule", it'd be neat if it included a way to re-serialize a DOM node back to HTML.
Basically, I'm using this library as a way to pre-process HTML before it's served to the user (an XML-flavoured templating engine, if you wish).
(I didn't find any obvious way to mark an issue as a suggestion/bug/..., hence why I added [suggestion] to the title. Did I miss anything? Github newbie here...)
/Users/owen/Documents/workspace/node-htmlparser/snippet.js:8
var handler = new htmlparser.DefaultHandler(function(err, dom) {
^
TypeError: undefined is not a function
at Object. (/Users/owen/Documents/workspace/node-htmlparser/snippet.js:8:15)
at Module._compile (module.js:449:26)
at Object.Module._extensions..js (module.js:467:10)
at Module.load (module.js:356:32)
at Function.Module._load (module.js:312:12)
at Module.runMain (module.js:492:10)
at process.startup.processNextTick.process._tickCallback (node.js:244:9)
Hi! Is it possible to synchronously parse a fully collected HTML, so that parser signature be the same to JSON.parse, require('qs').parse, require('querystring').parse?
TIA,
--Vladimir
If the parsed document has a <script> tag which is not closed, the parser acts like the document had nothing.
Example:
var
htmlparser = require('htmlparser'),
parser,
pHandler,
data = '<html><body><h1>Bla</h1><script src="somewhere"></body></html>';
pHandler = new htmlparser.DefaultHandler(function(err,doc){
if ( err )
throw err;
console.log("doc: ",doc);
});
The result is:
doc: []
The parser hangs with 100% cpu usage when parsing this file.
This can be reproduced by cloning https://gist.github.com/ed8a5b157d8644d68ca7.git and running bug.js.
Some old W3C pages were written in old style HTML. e.g. http://www.w3.org/TR/css3-2d-transforms/
node-htmlparser should be more forgiving.
version: 1.7.2
var request = require('request');
var jsdom = require('jsdom');
var url = 'http://www.w3.org/TR/css3-2d-transforms/';
request({uri:url}, function (error, response, body) {
var html = body;
var doc = jsdom.jsdom(html, null, {url: url});
console.log(doc.head+''); //[ HEAD ]
console.log(doc.body === null); //true
console.log(doc.head.childNodes[9].tagName); //BODY
});
I would like to convert DOM to JSON, modify the JSON and convet it back to DOM String.
Please suggest
node-htmlparser fails when the following snippet is present in the html document.
Clearly since the offending tag <SCR"+"IPT is inside of another <SCRIPT> tag node-htmlparser should not be trying to parse it.
The following regular expression used in parsing the HTML documents is vulnerable to ReDoS:
/(^\s+|\s+$)/g
The slowdown is moderately low: for 50.000 characters around 2.5 seconds matching time. However, I would still suggest one of the following:
If needed, I can provide an actual example showing the slowdown.
A declarative, efficient, and flexible JavaScript library for building user interfaces.
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
An Open Source Machine Learning Framework for Everyone
The Web framework for perfectionists with deadlines.
A PHP framework for web artisans
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
Some thing interesting about web. New door for the world.
A server is a program made to process requests and deliver data to clients.
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
Some thing interesting about visualization, use data art
Some thing interesting about game, make everyone happy.
We are working to build community through open source technology. NB: members must have two-factor auth.
Open source projects and samples from Microsoft.
Google ❤️ Open Source for everyone.
Alibaba Open Source for everyone
Data-Driven Documents codes.
China tencent open source team.