Awesome-Mirrors/awesome-docker
1
0
Fork 0
mirror of https://github.com/veggiemonk/awesome-docker.git synced 2025-01-03 11:41:14 -05:00
Code Issues Packages Projects Releases Wiki Activity

buildMetadata: clean up logs

Browse Source
This commit is contained in:
Julien Bisconti 2018-07-20 09:43:55 +02:00
parent be84a44da7
commit 19bf5ca215
No known key found for this signature in database
GPG Key ID: 62772C6698F736CB
3 changed files with 36 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.npmrc
View File

@ -1 +1 @@
save-exact=true save-exact=true

60
buildMetadata.js
View File

@ -4,18 +4,26 @@ const dayjs = require('dayjs');
require('draftlog').into(console); require('draftlog').into(console);
const LOG = {
error: (...args) => console.error('❌ ERROR', { ...args }),
debug: (...args) => {
if (process.env.DEBUG) console.log('💡 DEBUG: ', { ...args });
},
};
process.on('unhandledRejection', error => { process.on('unhandledRejection', error => {
console.log('unhandledRejection', error.message); LOG.error('unhandledRejection', error.message);
}); });
if (!process.env.TOKEN) { if (!process.env.TOKEN) {
console.log('❌ no credentials found.'); LOG.error('no credentials found.');
process.exit(1); process.exit(1);
} }
// --- ENV VAR --- // --- ENV VAR ---
const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 10; const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 10;
const DELAY = parseInt(process.env.DELAY, 10) || 3000; const DELAY = parseInt(process.env.DELAY, 10) || 3000;
// --- FILENAME --- // --- FILENAME ---
const README = 'README.md'; const README = 'README.md';
const DATA_FOLDER = 'data'; const DATA_FOLDER = 'data';
@ -24,6 +32,7 @@ const GITHUB_METADATA_FILE = `${DATA_FOLDER}/${dayjs().format(
)}-fetched_repo_data.json`; )}-fetched_repo_data.json`;
const LATEST_FILENAME = `${DATA_FOLDER}/latest`; const LATEST_FILENAME = `${DATA_FOLDER}/latest`;
const GITHUB_REPOS = `${DATA_FOLDER}/list_repos.json`; const GITHUB_REPOS = `${DATA_FOLDER}/list_repos.json`;
// --- HTTP --- // --- HTTP ---
const API = 'https://api.github.com/'; const API = 'https://api.github.com/';
const options = { const options = {
@ -38,12 +47,10 @@ const options = {
const removeHost = x => x.slice('https://github.com/'.length, x.length); const removeHost = x => x.slice('https://github.com/'.length, x.length);
const barLine = console.draft('Starting batch...'); const barLine = console.draft('Starting batch...');
const handleFailure = err => { const handleFailure = err => {
console.error('❌ ERROR', err); LOG.error(err);
process.exit(1); process.exit(1);
}; };
const LOG = (...logs) => {
if (process.env.DEBUG) console.log(...logs);
};
const delay = ms => const delay = ms =>
new Promise(resolve => { new Promise(resolve => {
setTimeout(() => resolve(), ms); setTimeout(() => resolve(), ms);
@ -86,9 +93,9 @@ async function batchFetchRepoMetadata(githubRepos) {
/* eslint-disable no-await-in-loop */ /* eslint-disable no-await-in-loop */
for (let i = 0; i < repos.length; i += BATCH_SIZE) { for (let i = 0; i < repos.length; i += BATCH_SIZE) {
const batch = repos.slice(i, i + BATCH_SIZE); const batch = repos.slice(i, i + BATCH_SIZE);
LOG({ batch }); LOG.debug({ batch });
const res = await fetchAll(batch); const res = await fetchAll(batch);
LOG('batch fetched...'); LOG.debug('batch fetched...');
metadata.push(...res); metadata.push(...res);
ProgressBar(i, BATCH_SIZE, repos.length); ProgressBar(i, BATCH_SIZE, repos.length);
// poor man's rate limiting so github don't ban us // poor man's rate limiting so github don't ban us
@ -99,7 +106,7 @@ async function batchFetchRepoMetadata(githubRepos) {
} }
function shouldUpdate(fileLatestUpdate) { function shouldUpdate(fileLatestUpdate) {
LOG({ fileLatestUpdate }); LOG.debug({ fileLatestUpdate });
if (!fileLatestUpdate) return true; if (!fileLatestUpdate) return true;
const hours = fileLatestUpdate.slice( const hours = fileLatestUpdate.slice(
@ -109,7 +116,9 @@ function shouldUpdate(fileLatestUpdate) {
const latestUpdate = dayjs( const latestUpdate = dayjs(
fileLatestUpdate.slice('data/'.length, 'data/YYYY-MM-DD'.length), fileLatestUpdate.slice('data/'.length, 'data/YYYY-MM-DD'.length),
).add(hours, 'hour'); ).add(hours, 'hour');
LOG({ latestUpdate: latestUpdate.format() });
LOG.debug({ latestUpdate: latestUpdate.format() });
const isMoreThanOneDay = dayjs().diff(latestUpdate, 'hours') >= 1; const isMoreThanOneDay = dayjs().diff(latestUpdate, 'hours') >= 1;
return isMoreThanOneDay; return isMoreThanOneDay;
} }
@ -117,37 +126,36 @@ function shouldUpdate(fileLatestUpdate) {
async function main() { async function main() {
try { try {
const getLatest = await fs.readFile(LATEST_FILENAME, 'utf8'); const getLatest = await fs.readFile(LATEST_FILENAME, 'utf8');
LOG('Checking if updating is needed');
LOG.debug('Checking if updating is needed');
if (!shouldUpdate(getLatest)) { if (!shouldUpdate(getLatest)) {
console.log('Last update was less than a day ago 😅. Exiting...'); LOG.debug('Last update was less than a day ago 😅. Exiting...');
process.exit(1); process.exit(1);
} }
const markdown = await fs.readFile(README, 'utf8'); const markdown = await fs.readFile(README, 'utf8');
const githubRepos = extractAllRepos(markdown); const githubRepos = extractAllRepos(markdown);
LOG('writing repo list to disk...'); LOG.debug('writing repo list to disk...');
await fs.outputJSON(GITHUB_REPOS, githubRepos, { spaces: 2 }); await fs.outputJSON(GITHUB_REPOS, githubRepos, { spaces: 2 });
LOG('fetching data...'); LOG.debug('fetching data...');
const metadata = await batchFetchRepoMetadata(githubRepos); const metadata = await batchFetchRepoMetadata(githubRepos);
LOG('writing metadata to disk...'); LOG.debug('writing metadata to disk...');
await fs.outputJSON(GITHUB_METADATA_FILE, metadata, { spaces: 2 }); await fs.outputJSON(GITHUB_METADATA_FILE, metadata, { spaces: 2 });
LOG('✅ metadata saved'); LOG.debug('✅ metadata saved');
LOG('removing latest...'); LOG.debug('removing latest...');
await fs.remove(LATEST_FILENAME); await fs.remove(LATEST_FILENAME);
LOG('writing latest...'); LOG.debug('writing latest...');
await fs.outputFile(LATEST_FILENAME, GITHUB_METADATA_FILE); await fs.outputFile(LATEST_FILENAME, GITHUB_METADATA_FILE);
LOG.debug('✅ late update time saved', {
LATEST_FILENAME,
GITHUB_METADATA_FILE,
});
if (process.env.DEBUG) { LOG.debug('gracefully shutting down.');
console.log('✅ late update time saved', {
LATEST_FILENAME,
GITHUB_METADATA_FILE,
});
}
console.log('gracefully shutting down.');
process.exit(); process.exit();
} catch (err) { } catch (err) {
handleFailure(err); handleFailure(err);

2
push.sh
View File

@ -19,4 +19,4 @@ git commit -m 'Automated update repository metadata'
echo 'Pushing changes' echo 'Pushing changes'
git push https://$GITHUB_USER:$GITHUB_TOKEN@github.com/veggiemonk/awesome-docker master >/dev/null 2>&1 git push https://$GITHUB_USER:$GITHUB_TOKEN@github.com/veggiemonk/awesome-docker master >/dev/null 2>&1
echo 'Done.' echo 'Done.'