-
Notifications
You must be signed in to change notification settings - Fork 0
/
crawl.js
89 lines (79 loc) · 2.27 KB
/
crawl.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
const { JSDOM } = require('jsdom')
async function crawlPage(baseURL, currentURL, pages){
// if this is an offsite URL, bail immediately
const currentUrlObj = new URL(currentURL)
const baseUrlObj = new URL(baseURL)
if (baseUrlObj.hostname !== currentUrlObj.hostname) {
console.error(
`given url is external url🫨: http://${currentUrlObj.hostname}${currentUrlObj.pathname}`,
);
}
const normalizedURL = normalizeURL(currentURL)
// if we've already visited this page
// just increase the count and don't repeat
// the http request
if (pages[normalizedURL] > 0){
pages[normalizedURL]++
return pages
}
// initialize this page in the map
// since it doesn't exist yet
pages[normalizedURL] = 1
// fetch and parse the html of the currentURL
console.log(`crawling ${currentURL}`)
let htmlBody = ''
try {
const resp = await fetch(currentURL)
if (resp.status > 399){
console.log(`Got HTTP error, status code: ${resp.status}`)
return pages
}
const contentType = resp.headers.get('content-type')
if (!contentType.includes('text/html')){
console.log(`Got non-html response: ${contentType}`)
return pages
}
htmlBody = await resp.text()
} catch (err){
console.log(err.message)
}
const nextURLs = getURLsFromHTML(htmlBody, baseURL)
for (const nextURL of nextURLs){
pages = await crawlPage(baseURL, nextURL, pages)
}
return pages
}
function getURLsFromHTML(htmlBody, baseURL){
const urls = []
const dom = new JSDOM(htmlBody)
const aElements = dom.window.document.querySelectorAll('a')
for (const aElement of aElements){
if (aElement.href.slice(0,1) === '/'){
try {
urls.push(new URL(aElement.href, baseURL).href)
} catch (err){
console.log(`${err.message}: ${aElement.href}`)
}
} else {
try {
urls.push(new URL(aElement.href).href)
} catch (err){
console.log(`${err.message}: ${aElement.href}`)
}
}
}
return urls
}
function normalizeURL(url){
const urlObj = new URL(url)
let fullPath = `${urlObj.host}${urlObj.pathname}`
if (fullPath.length > 0 && fullPath.slice(-1) === '/'){
fullPath = fullPath.slice(0, -1)
}
return fullPath
}
module.exports = {
crawlPage,
normalizeURL,
getURLsFromHTML
}