forked from alisson-barboza/crawler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
crawler.js
93 lines (80 loc) · 2.18 KB
/
crawler.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
const fs = require('fs')
const { Worker } = require('worker_threads')
const buffer = require('./buffer.js')
const readline = require('readline')
require('events').defaultMaxListeners = 10000
var readerQtd = 0
var downloaderQtd = 0
const urls = []
const filename = './tj.txt'
var reader = readline.createInterface({
input: fs.createReadStream(filename),
output: process.stdout,
console: false
});
reader.on('line', function (line) {
urls.push(line)
}).on('close', function (line) {
start(urls)
checkIfProgramEnded(4000)
});
async function start(urls) {
for (const url of urls) {
const reader = await getFreeReader();
setReader(reader, url)
}
}
function setReader(reader, msg) {
reader.postMessage(msg);
readerQtd--;
reader.on('message', async msg => {
await buffer.addLink(msg)
startDownloadingProcess()
});
}
async function checkIfProgramEnded(time) {
setTimeout(() => {
if (readerQtd === 0 && downloaderQtd === 0) {
console.log('Ending...')
setTimeout(() => {
process.exit()
}, 4000)
} else {
checkIfProgramEnded(time)
}
}, time);
}
async function startDownloadingProcess() {
var downloader = await getFreeDownloader()
if (downloader !== null) {
var link = await buffer.getLink()
if (link !== null) {
console.log('Downloading img from: ' + link);
setDownloader(downloader, link);
}
}
}
async function setDownloader(downloader, link) {
downloader.postMessage(link)
downloader.on('message', (msg) => {
downloaderQtd--
});
}
async function getFreeDownloader() {
console.log('Creating Downloader thread number: ' + downloaderQtd);
downloaderQtd++;
return createDownloader();
}
function createDownloader() {
// carai alisson, isso é muito seboso véi pqp kkk
return new Worker('./downloader.js');
}
async function getFreeReader() {
console.log('Creating Reader thread');
readerQtd ++;
return createReader();
}
function createReader() {
// carai alisson, isso é muito seboso véi pqp kkk
return new Worker('./reader.js');
}