Merge pull request #849 from veggiemonk/fix-build

Github Actions for PR (wip)

.eslintrc.js

@@ -1,22 +1,24 @@
 module.exports = {
   env: {
     browser: true,
-    node: true
+    node: true,
   },
   extends: [
     'airbnb-base',
     'plugin:import/errors',
     'plugin:import/warnings',
-    'prettier'
+    'prettier',
+    'eslint:recommended',
   ],
   plugins: ['import', 'prettier'],
   rules: {
+    camelcase: 0,
     'import/order': [
       'error',
       {
         groups: ['builtin', 'external', 'parent', 'sibling', 'index'],
-        'newlines-between': 'never'
+        'newlines-between': 'never',
-      }
+      },
     ],
     'no-console': 0,
     'prefer-template': 2,
@@ -24,8 +26,8 @@ module.exports = {
       'error',
       {
         singleQuote: true,
-        trailingComma: 'all'
+        trailingComma: 'all',
-      }
+      },
-    ]
+    ],
-  }
+  },
 };

.github/workflow/pull_request.yml

@@ -0,0 +1,30 @@
+name: Pull Requests
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@01aecccf739ca6ff86c0539fbc67a7a5007bbc81
+      - uses: actions/setup-node@83c9f7a7df54d6b57455f7c57ac414f2ae5fb8de
+        with:
+          node-version: 12
+
+      - uses: actions/cache@70655ec8323daeeaa7ef06d7c56e1b9191396cbe
+        id: cache
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Install Dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts --no-audit --no-progress --prefer-offline
+      - run: npm run test
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.travis.yml

@@ -1,7 +1,2 @@
-language: ruby
+language: node_js
-rvm:
+cache: npm
-  - 2.6.5
-before_script:
-  - gem install awesome_bot
-script:
-  - awesome_bot --allow-redirect --skip-save-results -w api.netlify.com,travis-ci.org,vimeo.com,github.com/sindresorhus,github.com/sindresorhus/awesome README.md

README.md

@@ -121,7 +121,7 @@ _Source:_ [What is Docker](https://www.docker.com/why-docker)
 - [Exploring ASP.NET Core with Docker in both Linux and Windows Containers](https://www.hanselman.com/blog/ExploringASPNETCoreWithDockerInBothLinuxAndWindowsContainers.aspx) Running ASP.NET Core apps in Linux and Windows containers, using [Docker for Windows][docker-for-windows]
 - [Running a Legacy ASP.NET App in a Windows Container](https://blog.sixeyed.com/dockerizing-nerd-dinner-part-1-running-a-legacy-asp-net-app-in-a-windows-container/) Steps for Dockerizing a legacy ASP.NET app and runnning as a Windows container
 - [Windows Containers and Docker: The 101](https://www.youtube.com/watch?v=N7SG2wEyQtM) :movie_camera: - A 20-minute overview, using Docker to run PowerShell, ASP.NET Core and ASP.NET apps
-- [Windows Containers Quick Start](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/index) Overview of Windows containers, drilling down to Quick Starts for Windows 10 and Windows Server 2016
+- [Windows Containers Quick Start](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/) Overview of Windows containers, drilling down to Quick Starts for Windows 10 and Windows Server 2016
 
 ---
 
@@ -660,7 +660,7 @@ Services to securely store your Docker images.
 
 ## Awesome Lists
 
-- [Awesome CI/CD](https://github.com/ciandcd/awesome-ciandcd) - Not specific to docker but relevant.
+- [Awesome CI/CD](https://github.com/cicdops/awesome-ciandcd) - Not specific to docker but relevant.
 - [Awesome Kubernetes](https://github.com/ramitsurana/awesome-kubernetes) by [@ramitsurana][ramitsurana]
 - [Awesome Linux Container](https://github.com/Friz-zy/awesome-linux-containers) more general about container than this repo, by [@Friz-zy](https://github.com/Friz-zy).
 - [Awesome Selfhosted](https://github.com/awesome-selfhosted/awesome-selfhosted) list of Free Software network services and web applications which can be hosted locally by running in a classical way (setup local web server and run applications from there) or in a Docker container. By [@Kickball](https://github.com/Kickball)

build.js

@@ -2,7 +2,6 @@ const fs = require('fs-extra');
 const cheerio = require('cheerio');
 const showdown = require('showdown');
 const Parcel = require('parcel-bundler');
-// const sm = require('sitemap');
 const { SitemapStream, streamToPromise } = require('sitemap');
 
 process.env.NODE_ENV = 'production';
@@ -13,7 +12,7 @@ const LOG = {
     if (process.env.DEBUG) console.log('💡 DEBUG: ', { ...args });
   },
 };
-const handleFailure = err => {
+const handleFailure = (err) => {
   LOG.error(err);
   process.exit(1);
 };
@@ -90,7 +89,7 @@ const bundle = () => {
       smStream.end();
       return streamToPromise(smStream);
     })
-    .then(sm =>
+    .then((sm) =>
       // Creates a sitemap object given the input configuration with URLs
       fs.outputFile(
         'dist/sitemap.xml',

exclude_in_test.json

@@ -0,0 +1,4 @@
+[
+    "https://vimeo.com",
+    "https://travis-ci.org/veggiemonk/awesome-docker.svg"
+]

old_build_beta.js

@@ -0,0 +1,132 @@
+const fs = require('fs-extra');
+const fetch = require('node-fetch');
+
+require('draftlog').into(console);
+
+const LOG = {
+  error: (...args) => console.error('  ERROR', { ...args }),
+  debug: (...args) => {
+    if (process.env.DEBUG) console.log('💡 DEBUG: ', { ...args });
+  },
+};
+const handleFailure = (err) => {
+  LOG.error(err);
+  process.exit(1);
+};
+
+process.on('unhandledRejection', handleFailure);
+
+if (!process.env.GITHUB_TOKEN) {
+  LOG.error('no credentials found.');
+  process.exit(1);
+}
+
+const TOKEN = process.env.GITHUB_TOKEN;
+
+// --- ENV VAR ---
+const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 10;
+const DELAY = parseInt(process.env.DELAY, 10) || 3000;
+const INTERVAL = parseInt(process.env.INTERVAL, 10) || 1;
+const INTERVAL_UNIT = process.env.INTERVAL_UNIT || 'days';
+
+// --- FILES ---
+const DATA_FOLDER = 'data';
+const README = 'README.md';
+const LATEST_FILENAME = `${DATA_FOLDER}/latest`;
+const GITHUB_REPOS = `${DATA_FOLDER}/repository.json`;
+const Authorization = `token ${TOKEN}`;
+
+// --- HTTP ---
+const API = 'https://api.github.com/';
+const options = {
+  method: 'GET',
+  headers: {
+    'User-Agent': 'awesome-docker script listing',
+    'Content-Type': 'application/json',
+    Authorization,
+  },
+};
+
+// ----------------------------------------------------------------------------
+const removeHost = (x) => x.slice('https://github.com/'.length, x.length);
+
+const delay = (ms) =>
+  new Promise((resolve) => {
+    setTimeout(() => resolve(), ms);
+  });
+
+const get = (pathURL, opt) => {
+  LOG.debug(`Fetching ${pathURL}`);
+  return fetch(`${API}repos/${pathURL}`, {
+    ...options,
+    ...opt,
+  })
+    .catch(handleFailure)
+    .then((response) => {
+      if (response.ok) return response.json();
+      throw new Error('Network response was not ok.');
+    })
+    .catch(handleFailure);
+};
+
+const fetchAll = (batch) =>
+  Promise.all(batch.map(async (pathURL) => get(pathURL)));
+
+const extractAllLinks = (markdown) => {
+  const re = /((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/g;
+  return markdown.match(re);
+};
+
+const extractAllRepos = (markdown) => {
+  const re = /https:\/\/github\.com\/([a-zA-Z0-9-._]+)\/([a-zA-Z0-9-._]+)/g;
+  const md = markdown.match(re);
+  return [...new Set(md)];
+};
+
+const ProgressBar = (i, batchSize, total) => {
+  const progress = Math.round((i / total) * 100);
+  const units = Math.round(progress / 2);
+  const barLine = console.draft('Starting batch...');
+  return barLine(
+    `[${'='.repeat(units)}${' '.repeat(50 - units)}] ${progress}%  -  # ${i}`,
+  );
+};
+// ----------------------------------------------------------------------------
+async function batchFetchRepoMetadata(githubRepos) {
+  const repos = githubRepos.map(removeHost);
+
+  const metadata = [];
+  /* eslint-disable no-await-in-loop */
+  for (let i = 0; i < repos.length; i += BATCH_SIZE) {
+    const batch = repos.slice(i, i + BATCH_SIZE);
+    LOG.debug({ batch });
+    const res = await fetchAll(batch);
+    LOG.debug('batch fetched...');
+    metadata.push(...res);
+    ProgressBar(i, BATCH_SIZE, repos.length);
+    // poor man's rate limiting so github doesn't ban us
+    await delay(DELAY);
+  }
+  ProgressBar(repos.length, BATCH_SIZE, repos.length);
+  return metadata;
+}
+
+async function main() {
+  try {
+    const markdown = await fs.readFile(README, 'utf8');
+    const links = extractAllLinks(markdown);
+    const githubRepos = extractAllRepos(markdown);
+    LOG.debug('writing repo list to disk...');
+    await fs.outputJSON(GITHUB_REPOS, githubRepos, { spaces: 2 });
+
+    LOG.debug('fetching data...');
+    const metadata = await batchFetchRepoMetadata(githubRepos);
+
+    LOG.debug('gracefully shutting down.');
+    process.exit();
+  } catch (err) {
+    handleFailure(err);
+  }
+}
+
+main();

package.json

@@ -4,7 +4,8 @@
   "description": "A curated list of Docker resources and projects Inspired by @sindresorhus and improved by amazing contributors",
   "main": "build.js",
   "scripts": {
-    "build": "rimraf ./dist/ && node build.js"
+    "build": "rimraf ./dist/ && node build.js",
+    "test": "node pull_request.js"
   },
   "repository": {
     "type": "git",
@@ -20,6 +21,7 @@
     "cheerio": "1.0.0-rc.3",
     "draftlog": "1.0.12",
     "fs-extra": "9.0.0",
+    "node-fetch": "2.6.0",
     "parcel-bundler": "1.12.4",
     "rimraf": "3.0.2",
     "showdown": "1.9.1",

pull_request.js

@@ -0,0 +1,207 @@
+const fs = require('fs-extra');
+const fetch = require('node-fetch');
+const exclude = require('./exclude_in_test.json');
+
+function envvar_undefined(variable_name) {
+  throw new Error(`${variable_name} must be defined`);
+}
+console.log({
+  DEBUG: process.env.DEBUG,
+});
+
+const README = 'README.md';
+const GITHUB_GQL_API = 'https://api.github.com/graphql';
+const TOKEN = process.env.GITHUB_TOKEN || envvar_undefined('GITHUB_TOKEN');
+
+const LINKS_OPTIONS = {
+  redirect: 'error',
+  headers: {
+    'Content-Type': 'application/json',
+    'user-agent':
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
+  },
+};
+
+const Authorization = `token ${TOKEN}`;
+
+const make_GQL_options = (query) => ({
+  method: 'POST',
+  headers: {
+    Authorization,
+    'Content-Type': 'application/json',
+    'user-agent':
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
+  },
+  body: JSON.stringify({ query }),
+});
+
+const LOG = {
+  error: (...args) => console.error('❌ ERROR', args),
+  error_string: (...args) =>
+    console.error('❌ ERROR', JSON.stringify({ ...args }, null, '  ')),
+  debug: (...args) => {
+    if (process.env.DEBUG) console.log('>>> DEBUG: ', { ...args });
+  },
+  debug_string: (...args) => {
+    if (process.env.DEBUG)
+      console.log('>>> DEBUG: ', JSON.stringify({ ...args }, null, '  '));
+  },
+};
+
+const handleFailure = (error) => {
+  console.error(`${error.message}: ${error.stack}`, { error });
+  process.exit(1);
+};
+
+process.on('unhandledRejection', handleFailure);
+
+const extract_all_links = (markdown) => {
+  // if you have a problem and you try to solve it with a regex,
+  // now you have two problems
+  // TODO: replace this mess with a mardown parser ?
+  const re = /(((https:(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/@\-\\\w]*))?)/g;
+  return markdown.match(re);
+};
+
+const find_duplicates = (arr) => {
+  const hm = {};
+  const dup = [];
+  arr.forEach((e) => {
+    if (hm[e]) dup.push(e);
+    else hm[e] = null;
+  });
+  return dup;
+};
+
+const partition = (arr, func) => {
+  const ap = [[], []];
+  arr.forEach((e) => (func(e) ? ap[0].push(e) : ap[1].push(e)));
+  return ap;
+};
+
+async function fetch_link(url) {
+  try {
+    const { ok, statusText, redirected } = await fetch(url, LINKS_OPTIONS);
+    return [url, { ok, status: statusText, redirected }];
+  } catch (error) {
+    return [url, { ok: false, status: error.message }];
+  }
+}
+
+async function batch_fetch({ arr, get, post_filter_func, BATCH_SIZE = 8 }) {
+  const result = [];
+  /* eslint-disable no-await-in-loop */
+  for (let i = 0; i < arr.length; i += BATCH_SIZE) {
+    const batch = arr.slice(i, i + BATCH_SIZE);
+    LOG.debug({ batch });
+    let res = await Promise.all(batch.map(get));
+    LOG.debug('batch fetched...');
+    res = post_filter_func ? res.filter(post_filter_func) : res;
+    LOG.debug_string({ res });
+    result.push(...res);
+  }
+  return result;
+}
+
+const extract_repos = (arr) =>
+  arr
+    .map((e) => e.substr('https://github.com/'.length).split('/'))
+    .filter((r) => r.length === 2 && r[1] !== '');
+
+const generate_GQL_query = (arr) =>
+  `query AWESOME_REPOS{ ${arr
+    .map(
+      ([owner, name]) =>
+        `repo_${owner.replace(/(-|\.)/g, '_')}_${name.replace(
+          /(-|\.)/g,
+          '_',
+        )}: repository(owner: "${owner}", name:"${name}"){ nameWithOwner } `,
+    )
+    .join('')} }`;
+
+// =============================================================
+// const batch_github_repos = async (github_links) => {
+//   const BATCH_SIZE = 50;
+//   const repos = extract_repos(github_links);
+//   for (let i = 0; i < repos.length; i += BATCH_SIZE) {
+//     const batch = repos.slice(i, i + BATCH_SIZE);
+//     const query = generate_GQL_query(batch);
+//     LOG.debug({ query });
+//     const gql_response = await fetch(
+//       'https://api.github.com/graphql',
+//       make_GQL_options(query),
+//     ).then((r) => r.json());
+//     LOG.debug({ gql_response });
+//   }
+// };
+// =============================================================
+
+const exclude_length = exclude.length;
+const exclude_from_list = (link) => {
+  let is_excluded = false;
+  for (let i = 0; i < exclude_length; i += 1) {
+    if (link.startsWith(exclude[i])) {
+      is_excluded = true;
+      break;
+    }
+  }
+  return is_excluded;
+};
+
+async function main() {
+  const has_error = {
+    show: false,
+    duplicates: '',
+    other_links_error: '',
+    github_repos: '',
+  };
+  const markdown = await fs.readFile(README, 'utf8');
+  let links = extract_all_links(markdown);
+  links = links.filter((l) => !exclude_from_list(l)); // exclude websites
+  LOG.debug_string({ links });
+  const duplicates = find_duplicates(links);
+  if (duplicates.length > 0) {
+    has_error.show = true;
+    has_error.duplicates = duplicates;
+  }
+  const [github_links, other_links] = partition(links, (link) =>
+    link.startsWith('https://github.com'),
+  );
+
+  const other_links_error = await batch_fetch({
+    arr: other_links,
+    get: fetch_link,
+    post_filter_func: (x) => !x[1].ok,
+    BATCH_SIZE: 8,
+  });
+  if (other_links_error.length > 0) {
+    has_error.show = true;
+    has_error.other_links_error = other_links_error;
+  }
+
+  const repos = extract_repos(github_links);
+  const query = generate_GQL_query(repos);
+  const options = make_GQL_options(query);
+  const gql_response = await fetch(GITHUB_GQL_API, options).then((r) =>
+    r.json(),
+  );
+
+  const { data } = gql_response;
+  if (gql_response.errors) {
+    has_error.show = true;
+    has_error.github_repos = gql_response.errors;
+  }
+  if (has_error.show) {
+    LOG.error_string(has_error);
+    process.exit(1);
+  }
+
+  // const repos_fetched = Object.entries(data)
+  //   .map(([, /* k , */ v]) => v.nameWithOwner)
+  //   .sort((a, b) => b - a);
+
+  console.log({ repos_fetched: data.length });
+}
+
+console.log('starting...');
+main();

renovate.json

@@ -1,5 +0,0 @@
-{
-  "extends": [
-    "config:base"
-  ]
-}