awesome-docker/pull_request.js

const fs = require('fs-extra');
const fetch = require('node-fetch');
const exclude = require('./exclude_in_test.json');

function envvar_undefined(variable_name) {
  throw new Error(`${variable_name} must be defined`);
}
console.log({
  DEBUG: process.env.DEBUG || false,
});

const README = 'README.md';
const GITHUB_GQL_API = 'https://api.github.com/graphql';
const TOKEN = process.env.GITHUB_TOKEN || envvar_undefined('GITHUB_TOKEN');

const LINKS_OPTIONS = {
  redirect: 'error',
  headers: {
    'Content-Type': 'application/json',
    'user-agent':
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
  },
};

const Authorization = `token ${TOKEN}`;

const make_GQL_options = (query) => ({
  method: 'POST',
  headers: {
    Authorization,
    'Content-Type': 'application/json',
    'user-agent':
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
  },
  body: JSON.stringify({ query }),
});

const LOG = {
  error: (...args) => console.error('❌ ERROR', args),
  error_string: (...args) =>
    console.error('❌ ERROR', JSON.stringify({ ...args }, null, '  ')),
  debug: (...args) => {
    if (process.env.DEBUG) console.log('>>> DEBUG: ', { ...args });
  },
  debug_string: (...args) => {
    if (process.env.DEBUG)
      console.log('>>> DEBUG: ', JSON.stringify({ ...args }, null, '  '));
  },
};

const handleFailure = (error) => {
  console.error(`${error.message}: ${error.stack}`, { error });
  process.exit(1);
};

process.on('unhandledRejection', handleFailure);

const extract_all_links = (markdown) => {
  // if you have a problem and you try to solve it with a regex,
  // now you have two problems
  // TODO: replace this mess with a mardown parser ?
  const re = /(((https:(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/@\-\\\w]*))?)/g;
  return markdown.match(re);
};

const find_duplicates = (arr) => {
  const hm = {};
  const dup = [];
  arr.forEach((e) => {
    if (hm[e]) dup.push(e);
    else hm[e] = true;
  });
  return dup;
};

const partition = (arr, func) => {
  const ap = [[], []];
  arr.forEach((e) => (func(e) ? ap[0].push(e) : ap[1].push(e)));
  return ap;
};

async function fetch_link(url) {
  try {
    const { ok, statusText, redirected } = await fetch(url, LINKS_OPTIONS);
    return [url, { ok, status: statusText, redirected }];
  } catch (error) {
    return [url, { ok: false, status: error.message }];
  }
}

async function batch_fetch({ arr, get, post_filter_func, BATCH_SIZE = 8 }) {
  const result = [];
  /* eslint-disable no-await-in-loop */
  for (let i = 0; i < arr.length; i += BATCH_SIZE) {
    const batch = arr.slice(i, i + BATCH_SIZE);
    LOG.debug_string({ batch });
    let res = await Promise.all(batch.map(get));
    console.log(`batch fetched...${i + BATCH_SIZE}`);
    res = post_filter_func ? res.filter(post_filter_func) : res;
    LOG.debug_string({ res });
    result.push(...res);
  }
  return result;
}

const extract_repos = (arr) =>
  arr
    .map((e) => e.substr('https://github.com/'.length).split('/'))
    .filter((r) => r.length === 2 && r[1] !== '');

const generate_GQL_query = (arr) =>
  `query AWESOME_REPOS{ ${arr
    .map(
      ([owner, name]) =>
        `repo_${owner.replace(/(-|\.)/g, '_')}_${name.replace(
          /(-|\.)/g,
          '_',
        )}: repository(owner: "${owner}", name:"${name}"){ nameWithOwner } `,
    )
    .join('')} }`;

// =============================================================
// const batch_github_repos = async (github_links) => {
//   const BATCH_SIZE = 50;
//   const repos = extract_repos(github_links);
//   for (let i = 0; i < repos.length; i += BATCH_SIZE) {
//     const batch = repos.slice(i, i + BATCH_SIZE);
//     const query = generate_GQL_query(batch);
//     LOG.debug({ query });
//     const gql_response = await fetch(
//       'https://api.github.com/graphql',
//       make_GQL_options(query),
//     ).then((r) => r.json());
//     LOG.debug({ gql_response });
//   }
// };
// =============================================================

const exclude_length = exclude.length;
const exclude_from_list = (link) => {
  let is_excluded = false;
  for (let i = 0; i < exclude_length; i += 1) {
    if (link.startsWith(exclude[i])) {
      is_excluded = true;
      break;
    }
  }
  return is_excluded;
};

async function main() {
  const has_error = {
    show: false,
    duplicates: '',
    other_links_error: '',
    github_repos: '',
  };
  const markdown = await fs.readFile(README, 'utf8');
  let links = extract_all_links(markdown);
  links = links.filter((l) => !exclude_from_list(l)); // exclude websites
  LOG.debug_string({ links });

  console.log(`total links to check ${links.length}`);

  console.log('checking for duplicates links...');

  const duplicates = find_duplicates(links);
  if (duplicates.length > 0) {
    has_error.show = true;
    has_error.duplicates = duplicates;
  }
  LOG.debug_string({ duplicates });
  const [github_links, external_links] = partition(links, (link) =>
    link.startsWith('https://github.com'),
  );

  console.log(`checking ${external_links.length} external links...`);

  const external_links_error = await batch_fetch({
    arr: external_links,
    get: fetch_link,
    post_filter_func: (x) => !x[1].ok,
    BATCH_SIZE: 8,
  });
  if (external_links_error.length > 0) {
    has_error.show = true;
    has_error.other_links_error = external_links_error;
  }

  console.log(`checking ${github_links.length} GitHub repositories...`);

  const repos = extract_repos(github_links);
  const query = generate_GQL_query(repos);
  const options = make_GQL_options(query);
  const gql_response = await fetch(GITHUB_GQL_API, options).then((r) =>
    r.json(),
  );
  if (gql_response.errors) {
    has_error.show = true;
    has_error.github_repos = gql_response.errors;
  }

  console.log({
    TEST_PASSED: has_error.show,
    GITHUB_REPOSITORY: github_links.length,
    EXTERNAL_LINKS: external_links.length,
  });

  if (has_error.show) {
    LOG.error_string(has_error);
    process.exit(1);
  }
}

console.log('starting...');
main();