107 lines
3.0 KiB
Markdown
107 lines
3.0 KiB
Markdown
import fetch from 'node-fetch';
|
|
import { isRedirect } from 'node-fetch';
|
|
import {readFileSync} from 'fs';
|
|
|
|
const LINKS_OPTIONS = {
|
|
redirect: 'manual',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
'user-agent':
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
|
|
},
|
|
};
|
|
|
|
const LOG = {
|
|
error: (...args) => console.error('❌ ERROR', args),
|
|
error_string: (...args) =>
|
|
console.error('❌ ERROR', JSON.stringify({ ...args }, null, ' ')),
|
|
debug: (...args) => {
|
|
if (process.env.DEBUG) console.log('>>> DEBUG: ', { ...args });
|
|
},
|
|
debug_string: (...args) => {
|
|
if (process.env.DEBUG)
|
|
console.log('>>> DEBUG: ', JSON.stringify({ ...args }, null, ' '));
|
|
},
|
|
};
|
|
|
|
const handleFailure = (error) => {
|
|
console.error(`${error.message}: ${error.stack}`, { error });
|
|
process.exit(1);
|
|
};
|
|
|
|
process.on('unhandledRejection', handleFailure);
|
|
|
|
const extract_all_links = (markdown) => {
|
|
// if you have a problem and you try to solve it with a regex,
|
|
// now you have two problems
|
|
// TODO: replace this mess with a mardown parser ?
|
|
const re = /(((https:(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/@\-\\\w]*))?)/g;
|
|
return markdown.match(re);
|
|
};
|
|
|
|
const find_duplicates = (arr) => {
|
|
const hm = {};
|
|
const dup = [];
|
|
arr.forEach((e) => {
|
|
if (hm[e]) dup.push(e);
|
|
else hm[e] = true;
|
|
});
|
|
return dup;
|
|
};
|
|
|
|
const partition = (arr, func) => {
|
|
const ap = [[], []];
|
|
arr.forEach((e) => (func(e) ? ap[0].push(e) : ap[1].push(e)));
|
|
return ap;
|
|
};
|
|
|
|
async function fetch_link(url) {
|
|
try {
|
|
const { headers, ok, status, statusText } = await fetch(url, LINKS_OPTIONS);
|
|
const redirect = isRedirect(status) ? { redirect: { src: url, dst: headers.get("location") } } : {};
|
|
return [url, { ok, status: statusText, ...redirect }];
|
|
} catch (error) {
|
|
return [url, { ok: false, status: error.message }];
|
|
}
|
|
}
|
|
|
|
async function batch_fetch({ arr, get, post_filter_func, BATCH_SIZE = 8 }) {
|
|
const result = [];
|
|
/* eslint-disable no-await-in-loop */
|
|
for (let i = 0; i < arr.length; i += BATCH_SIZE) {
|
|
const batch = arr.slice(i, i + BATCH_SIZE);
|
|
LOG.debug_string({ batch });
|
|
let res = await Promise.all(batch.map(get));
|
|
console.log(`batch fetched...${i + BATCH_SIZE}`);
|
|
res = post_filter_func ? res.filter(post_filter_func) : res;
|
|
LOG.debug_string({ res });
|
|
result.push(...res);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
const data = readFileSync('./tests/exclude_in_test.json')
|
|
const exclude = JSON.parse(data)
|
|
const exclude_length = exclude.length;
|
|
const exclude_from_list = (link) => {
|
|
let is_excluded = false;
|
|
for (let i = 0; i < exclude_length; i += 1) {
|
|
if (link.startsWith(exclude[i])) {
|
|
is_excluded = true;
|
|
break;
|
|
}
|
|
}
|
|
return is_excluded;
|
|
};
|
|
|
|
export default {
|
|
LOG,
|
|
handleFailure,
|
|
extract_all_links,
|
|
find_duplicates,
|
|
partition,
|
|
fetch_link,
|
|
batch_fetch,
|
|
exclude_from_list,
|
|
};
|