diff --git a/.gitignore b/.gitignore
index cc2ae1e..0825698 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,7 +6,7 @@ node_modules
.cache
dist
package-lock.json
-website/index.merged.html
website/index.html
+website/table.html
**/.DS_Store
diff --git a/buildMetadata.js b/buildMetadata.js
index 70a087b..659c745 100644
--- a/buildMetadata.js
+++ b/buildMetadata.js
@@ -22,7 +22,7 @@ const GITHUB_METADATA_FILE = `data/${dayjs().format(
'YYYY-MM-DDTHH.mm.ss',
)}-fetched_repo_data.json`;
const GITHUB_REPOS = 'data/list_repos.json';
-const METADATA_TABLE = 'data/table.md';
+// const METADATA_TABLE = 'data/table.md';
// --- HTTP ---
const API = 'https://api.github.com/';
const options = {
@@ -73,41 +73,41 @@ const ProgressBar = (i, batchSize, total) => {
);
};
-const getLastUpdate = updated => {
- const updt = Number(dayjs(updated).diff(dayjs(), 'days'));
- if (updt < 0) {
- if (Math.abs(updt) === 1) return `1 day ago`;
- return `${Math.abs(updt)} days ago`;
- } else if (updt === 0) return 'today';
- return updated;
-};
+// const getLastUpdate = updated => {
+// const updt = Number(dayjs(updated).diff(dayjs(), 'days'));
+// if (updt < 0) {
+// if (Math.abs(updt) === 1) return `1 day ago`;
+// return `${Math.abs(updt)} days ago`;
+// } else if (updt === 0) return 'today';
+// return updated;
+// };
-const createLine = data => {
- const {
- name,
- html_url: repoURL,
- description,
- homepage,
- stargazers_count: stargazers,
- updated_at: updated,
- language,
- license,
- owner,
- } = data;
- if (!data || !name) return '|ERROR |';
+// const createLine = data => {
+// const {
+// name,
+// html_url: repoURL,
+// description,
+// homepage,
+// stargazers_count: stargazers,
+// updated_at: updated,
+// language,
+// license,
+// owner,
+// } = data;
+// if (!data || !name) return '|ERROR |';
- const lineData = [
- `[${name}](${repoURL})`,
- description || '-',
- homepage || '-',
- stargazers,
- getLastUpdate(updated),
- language,
- license && `[${license.name}](${license.url})`,
- owner && `[${owner.login}](${owner.html_url})`,
- ];
- return `|${lineData.join('|')}|`;
-};
+// const lineData = [
+// `[${name}](${repoURL})`,
+// description || '-',
+// homepage || '-',
+// stargazers,
+// getLastUpdate(updated),
+// language,
+// license && `[${license.name}](${license.url})`,
+// owner && `[${owner.login}](${owner.html_url})`,
+// ];
+// return `|${lineData.join('|')}|`;
+// };
async function batchFetchRepoMetadata(githubRepos) {
const repos = githubRepos.map(removeHost);
@@ -118,37 +118,31 @@ async function batchFetchRepoMetadata(githubRepos) {
const batch = repos.slice(i, i + BATCH_SIZE);
if (process.env.DEBUG) console.log({ batch });
const res = await fetchAll(batch);
- fs.appendFile(
- GITHUB_METADATA_FILE,
- JSON.stringify(res, null, 2),
- printError,
- );
metadata.push(...res);
ProgressBar(i, BATCH_SIZE, repos.length);
- // poor's man rate limiting so github don't blacklist us
- // also the file has the time to append the data
+ // poor man's rate limiting so github don't ban us
await delay(DELAY);
}
ProgressBar(repos.length, BATCH_SIZE, repos.length);
return metadata;
}
-const convertToTable = data => {
- const header = `
-# Repository Metadata
+// const convertToTable = data => {
+// const header = `
+// # Repository Metadata
-| Name | Description | Homepage | Star | Updated | Language | License | Author |
-| ----------- | ----------- | -------- | ---- | ------- | -------- | :---: | ------:|`;
- const table = [header]
- .concat(
- data
- .sort((a, b) => Number(b.stargazers_count) - Number(a.stargazers_count))
- .map(createLine),
- )
- .join('\n');
+// | Name | Description | Homepage | Star | Updated | Language | License | Author |
+// | ----------- | ----------- | -------- | ---- | ------- | -------- | :---: | ------:|`;
+// const table = [header]
+// .concat(
+// data
+// .sort((a, b) => Number(b.stargazers_count) - Number(a.stargazers_count))
+// .map(createLine),
+// )
+// .join('\n');
- return writeFile(METADATA_TABLE, table);
-};
+// return writeFile(METADATA_TABLE, table);
+// };
async function main() {
try {
@@ -162,11 +156,15 @@ async function main() {
const metadata = batchFetchRepoMetadata(githubRepos);
- console.log('✅ fetching metadata');
+ await writeFile(
+ GITHUB_METADATA_FILE,
+ JSON.stringify(metadata, null, 2),
+ printError,
+ );
+ console.log('✅ metadata saved');
- await convertToTable(metadata);
-
- console.log('✅ writing metadata table');
+ // await convertToTable(metadata);
+ // console.log('✅ writing metadata table');
} catch (err) {
printError(err);
}
diff --git a/buildTable.js b/buildTable.js
new file mode 100644
index 0000000..f1aa296
--- /dev/null
+++ b/buildTable.js
@@ -0,0 +1,106 @@
+const fs = require('fs');
+// const showdown = require('showdown');
+const cheerio = require('cheerio');
+// const Parcel = require('parcel-bundler');
+const dayjs = require('dayjs');
+const metaData = require('./data/2018-06-06T17.54.30-fetched_repo_data.json');
+
+process.env.NODE_ENV = 'production';
+
+process.on('unhandledRejection', error => {
+ console.log('unhandledRejection', error.message);
+});
+
+// const table = 'data/table.md';
+const templateHTML = 'website/table.tmpl.html';
+// const merged = 'website/table.html';
+const destination = 'website/table.html';
+
+const valueNames = [
+ 'name',
+ 'description',
+ 'homepage',
+ 'star',
+ 'updated',
+ 'language',
+ 'license',
+ 'author',
+];
+
+const getLastUpdate = updated => {
+ const updt = Number(dayjs(updated).diff(dayjs(), 'days'));
+ if (updt < 0) {
+ if (Math.abs(updt) === 1) return `1 day ago`;
+ return `${Math.abs(updt)} days ago`;
+ } else if (updt === 0) return 'today';
+ return updated;
+};
+
+const formatEntry = (
+ {
+ name,
+ html_url: repoURL,
+ description,
+ homepage,
+ stargazers_count: stargazers,
+ updated_at: updated,
+ language,
+ license,
+ owner,
+ },
+ i,
+) =>
+ [
+ `
`,
+ `${name}`,
+ `${description || '-'}
`,
+ (homepage &&
+ `🔗 website`) ||
+ '',
+ `⭐️${stargazers}
`,
+ `${getLastUpdate(updated)}
`,
+ (language && `💻${language}
`) || '',
+ (license &&
+ `📃${
+ license.name
+ }`) ||
+ '',
+ owner &&
+ `Made by ${
+ owner.login
+ }`,
+ '',
+ ].join('');
+
+function main() {
+ const indexTemplate = fs.readFileSync(templateHTML, 'utf8');
+ const $ = cheerio.load(indexTemplate);
+ const btn = valueNames.map(
+ v => ``,
+ );
+ $('#md').append(
+ [
+ ` ${btn.join('')}`,
+ '',
+ metaData.map(formatEntry).join(''),
+ '
',
+ ].join(''),
+ );
+
+ $('body').append(
+ '',
+ );
+
+ $('body').append(`
+
+ `);
+ console.log('Writing table.html');
+ fs.writeFileSync(destination, $.html(), 'utf8');
+ console.log('DONE 👍');
+}
+
+main();
diff --git a/data/2018-06-06T17.54.30-fetched_repo_data.json b/data/2018-06-06T17.54.30-fetched_repo_data.json
index 3bfd5c9..58a2ed6 100644
--- a/data/2018-06-06T17.54.30-fetched_repo_data.json
+++ b/data/2018-06-06T17.54.30-fetched_repo_data.json
@@ -1123,9 +1123,7 @@
},
"network_count": 1333,
"subscribers_count": 300
- }
-][
- {
+},{
"id": 19360616,
"node_id": "MDEwOlJlcG9zaXRvcnkxOTM2MDYxNg==",
"name": "swarm",
@@ -2226,9 +2224,7 @@
},
"network_count": 7,
"subscribers_count": 4
- }
-][
- {
+},{
"id": 67536560,
"node_id": "MDEwOlJlcG9zaXRvcnk2NzUzNjU2MA==",
"name": "elsy",
@@ -3467,9 +3463,7 @@
},
"network_count": 65,
"subscribers_count": 40
- }
-][
- {
+},{
"id": 27701139,
"node_id": "MDEwOlJlcG9zaXRvcnkyNzcwMTEzOQ==",
"name": "conduit",
@@ -4808,9 +4802,7 @@
},
"network_count": 9,
"subscribers_count": 4
- }
-][
- {
+},{
"id": 85515090,
"node_id": "MDEwOlJlcG9zaXRvcnk4NTUxNTA5MA==",
"name": "docker-flow-monitor",
@@ -5977,9 +5969,7 @@
},
"network_count": 11,
"subscribers_count": 6
- }
-][
- {
+},{
"id": 21704134,
"node_id": "MDEwOlJlcG9zaXRvcnkyMTcwNDEzNA==",
"name": "flannel",
@@ -7378,9 +7368,7 @@
},
"network_count": 31,
"subscribers_count": 10
- }
-][
- {
+},{
"id": 72737874,
"node_id": "MDEwOlJlcG9zaXRvcnk3MjczNzg3NA==",
"name": "haven-platform",
@@ -8639,9 +8627,7 @@
},
"network_count": 152,
"subscribers_count": 88
- }
-][
- {
+},{
"id": 26337322,
"node_id": "MDEwOlJlcG9zaXRvcnkyNjMzNzMyMg==",
"name": "rancher",
@@ -9854,9 +9840,7 @@
},
"network_count": 363,
"subscribers_count": 205
- }
-][
- {
+},{
"id": 52381034,
"node_id": "MDEwOlJlcG9zaXRvcnk1MjM4MTAzNA==",
"name": "workflow",
@@ -11167,9 +11151,7 @@
},
"network_count": 2,
"subscribers_count": 2
- }
-][
- {
+},{
"id": 42408804,
"node_id": "MDEwOlJlcG9zaXRvcnk0MjQwODgwNA==",
"name": "traefik",
@@ -12408,9 +12390,7 @@
},
"network_count": 122,
"subscribers_count": 52
- }
-][
- {
+},{
"id": 49986046,
"node_id": "MDEwOlJlcG9zaXRvcnk0OTk4NjA0Ng==",
"name": "falco",
@@ -13629,9 +13609,7 @@
},
"network_count": 98,
"subscribers_count": 31
- }
-][
- {
+},{
"id": 34205411,
"node_id": "MDEwOlJlcG9zaXRvcnkzNDIwNTQxMQ==",
"name": "docker-unison",
@@ -14772,9 +14750,7 @@
},
"network_count": 31,
"subscribers_count": 19
- }
-][
- {
+},{
"id": 77419377,
"node_id": "MDEwOlJlcG9zaXRvcnk3NzQxOTM3Nw==",
"name": "ctop",
@@ -15887,9 +15863,7 @@
},
"network_count": 41,
"subscribers_count": 36
- }
-][
- {
+},{
"id": 42531199,
"node_id": "MDEwOlJlcG9zaXRvcnk0MjUzMTE5OQ==",
"name": "dvm",
@@ -17002,9 +16976,7 @@
},
"network_count": 1,
"subscribers_count": 3
- }
-][
- {
+},{
"id": 34074446,
"node_id": "MDEwOlJlcG9zaXRvcnkzNDA3NDQ0Ng==",
"name": "wharfee",
@@ -18137,9 +18109,7 @@
},
"network_count": 369,
"subscribers_count": 109
- }
-][
- {
+},{
"id": 51555295,
"node_id": "MDEwOlJlcG9zaXRvcnk1MTU1NTI5NQ==",
"name": "rapid",
@@ -19212,9 +19182,7 @@
},
"network_count": 4,
"subscribers_count": 9
- }
-][
- {
+},{
"id": 100341708,
"node_id": "MDEwOlJlcG9zaXRvcnkxMDAzNDE3MDg=",
"name": "lstags",
@@ -20387,9 +20355,7 @@
},
"network_count": 36,
"subscribers_count": 26
- }
-][
- {
+},{
"id": 39589297,
"node_id": "MDEwOlJlcG9zaXRvcnkzOTU4OTI5Nw==",
"name": "dlayer",
@@ -21582,9 +21548,7 @@
},
"network_count": 16,
"subscribers_count": 43
- }
-][
- {
+},{
"id": 36849312,
"node_id": "MDEwOlJlcG9zaXRvcnkzNjg0OTMxMg==",
"name": "runlike",
@@ -22659,9 +22623,7 @@
},
"network_count": 399,
"subscribers_count": 52
- }
-][
- {
+},{
"id": 49360444,
"node_id": "MDEwOlJlcG9zaXRvcnk0OTM2MDQ0NA==",
"name": "Dockerfiles",
@@ -23802,9 +23764,7 @@
},
"network_count": 5,
"subscribers_count": 5
- }
-][
- {
+},{
"id": 29760781,
"node_id": "MDEwOlJlcG9zaXRvcnkyOTc2MDc4MQ==",
"name": "docket",
@@ -25003,9 +24963,7 @@
},
"network_count": 139,
"subscribers_count": 103
- }
-][
- {
+},{
"id": 12467623,
"node_id": "MDEwOlJlcG9zaXRvcnkxMjQ2NzYyMw==",
"name": "dockerode",
@@ -26164,9 +26122,7 @@
},
"network_count": 246,
"subscribers_count": 56
- }
-][
- {
+},{
"id": 16607898,
"node_id": "MDEwOlJlcG9zaXRvcnkxNjYwNzg5OA==",
"name": "drone",
@@ -27365,9 +27321,7 @@
},
"network_count": 4,
"subscribers_count": 1
- }
-][
- {
+},{
"id": 18931097,
"node_id": "MDEwOlJlcG9zaXRvcnkxODkzMTA5Nw==",
"name": "devstep",
@@ -28408,9 +28362,7 @@
},
"network_count": 9,
"subscribers_count": 11
- }
-][
- {
+},{
"id": 78498440,
"node_id": "MDEwOlJlcG9zaXRvcnk3ODQ5ODQ0MA==",
"name": "lando",
@@ -29603,9 +29555,7 @@
},
"network_count": 618,
"subscribers_count": 225
- }
-][
- {
+},{
"id": 59715467,
"node_id": "MDEwOlJlcG9zaXRvcnk1OTcxNTQ2Nw==",
"name": "docker-lambda",
@@ -30778,9 +30728,7 @@
},
"network_count": 65,
"subscribers_count": 40
- }
-][
- {
+},{
"id": 13003799,
"node_id": "MDEwOlJlcG9zaXRvcnkxMzAwMzc5OQ==",
"name": "azk",
@@ -31947,9 +31895,7 @@
},
"network_count": 36,
"subscribers_count": 12
- }
-][
- {
+},{
"id": 72878982,
"node_id": "MDEwOlJlcG9zaXRvcnk3Mjg3ODk4Mg==",
"name": "landscape",
@@ -33242,9 +33188,7 @@
},
"network_count": 66,
"subscribers_count": 69
- }
-][
- {
+},{
"id": 30508772,
"node_id": "MDEwOlJlcG9zaXRvcnkzMDUwODc3Mg==",
"name": "Docker",
@@ -34399,9 +34343,7 @@
},
"network_count": 343,
"subscribers_count": 167
- }
-][
- {
+},{
"id": 23929024,
"node_id": "MDEwOlJlcG9zaXRvcnkyMzkyOTAyNA==",
"name": "dockerfiles",
diff --git a/fetchRepos.js b/fetchRepos.js
index 0de374b..acd87a3 100644
--- a/fetchRepos.js
+++ b/fetchRepos.js
@@ -85,19 +85,22 @@ async function main() {
const repos = githubRepos.map(removeHost);
+ const metadata = [];
/* eslint-disable no-await-in-loop */
for (let i = 0; i < repos.length; i += BATCH_SIZE) {
const batch = repos.slice(i, i + BATCH_SIZE);
if (process.env.DEBUG) console.log({ batch });
const res = await Promise.all(batch.map(async path => get(path)));
- fs.appendFile(
- GITHUB_METADATA_FILE,
- JSON.stringify(res, null, 2),
- err => err && console.error(err),
- );
+ metadata.push(...res);
ProgressBar(i, BATCH_SIZE, repos.length);
await delay(DELAY);
}
+
+ fs.writeFile(
+ GITHUB_METADATA_FILE,
+ JSON.stringify(metadata, null, 2),
+ err => err && console.error(err),
+ );
ProgressBar(repos.length, BATCH_SIZE, repos.length);
} catch (err) {
console.error('ERROR:', err);
diff --git a/package.json b/package.json
index 2611e52..34050cb 100644
--- a/package.json
+++ b/package.json
@@ -23,9 +23,11 @@
"homepage": "https://github.com/veggiemonk/awesome-docker#readme",
"dependencies": {
"cheerio": "^1.0.0-rc.2",
- "critical": "^1.3.2",
- "dayjs": "^1.6.4",
+ "critical": "^1.3.3",
+ "dayjs": "^1.6.6",
"draftlog": "^1.0.12",
+ "jsdom": "^11.11.0",
+ "list.js": "^1.5.0",
"node-fetch": "^2.1.2",
"parcel-bundler": "^1.8.1",
"rimraf": "^2.6.2",
@@ -39,6 +41,6 @@
"eslint-plugin-import": "^2.12.0",
"eslint-plugin-jsx-a11y": "^6.0.3",
"eslint-plugin-prettier": "^2.6.0",
- "prettier": "^1.13.0"
+ "prettier": "^1.13.4"
}
}
diff --git a/website/index.js b/website/index.js
index 28f3ac5..c96e45f 100644
--- a/website/index.js
+++ b/website/index.js
@@ -1,3 +1,5 @@
+const list = require('list.js');
+
const main = () => {
console.log('hi!');
};
diff --git a/website/table.tmpl.html b/website/table.tmpl.html
new file mode 100644
index 0000000..8b0843e
--- /dev/null
+++ b/website/table.tmpl.html
@@ -0,0 +1,257 @@
+
+
+
+
+
+
+
+ Awesome-docker
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file