mirror of
https://github.com/richgel999/ufo_data.git
synced 2024-09-20 08:25:43 +00:00
299 lines
9.1 KiB
C++
299 lines
9.1 KiB
C++
// ufojson.cpp
|
|
// Copyright (C) 2023 Richard Geldreich, Jr.
|
|
|
|
#include "utils.h"
|
|
#include "markdown_proc.h"
|
|
#include "ufojson_core.h"
|
|
#include "udb.h"
|
|
#include "converters.h"
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
static void detect_bad_urls()
|
|
{
|
|
string_vec unique_urls;
|
|
bool utf8_flag = false;
|
|
if (!read_text_file("unique_urls.txt", unique_urls, utf8_flag))
|
|
panic("Can't read file unique_urls.txt");
|
|
|
|
uint32_t total_processed = 0;
|
|
|
|
string_vec failed_urls;
|
|
for (const auto& url : unique_urls)
|
|
{
|
|
printf("-------------- URL : %u\n", total_processed);
|
|
|
|
string_vec reply;
|
|
bool status = invoke_curl(url, reply);
|
|
bool not_found = false;
|
|
|
|
if (status)
|
|
{
|
|
for (auto str : reply)
|
|
{
|
|
str = string_lower(str);
|
|
|
|
if ((string_find_first(str, "404 not found") != -1) ||
|
|
(string_find_first(str, "cannot find the requested page") != -1))
|
|
{
|
|
not_found = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if ((!status) || (not_found))
|
|
{
|
|
failed_urls.push_back(url);
|
|
printf("FAILED: %s\n", url.c_str());
|
|
}
|
|
else
|
|
{
|
|
printf("SUCCEEDED: %s\n", url.c_str());
|
|
}
|
|
|
|
total_processed++;
|
|
if ((total_processed % 25) == 24)
|
|
{
|
|
if (!write_text_file("failed_urls.txt", failed_urls, false))
|
|
panic("Unable to create file failed_urs.txt");
|
|
}
|
|
}
|
|
|
|
printf("Total urls: %zu, failed: %zu\n", unique_urls.size(), failed_urls.size());
|
|
|
|
if (!write_text_file("failed_urls.txt", failed_urls, false))
|
|
panic("Unable to create file failed_urs.txt");
|
|
}
|
|
|
|
// Windows defaults to code page 437:
|
|
// https://www.ascii-codes.com/
|
|
// We want code page 1252
|
|
// http://www.alanwood.net/demos/ansi.html
|
|
|
|
// Main code
|
|
int wmain(int argc, wchar_t* argv[])
|
|
{
|
|
assert(cTotalPrefixes == sizeof(g_date_prefix_strings) / sizeof(g_date_prefix_strings[0]));
|
|
|
|
string_vec args;
|
|
convert_args_to_utf8(args, argc, argv);
|
|
|
|
// Set ANSI Latin 1; Western European (Windows) code page for output.
|
|
SetConsoleOutputCP(1252);
|
|
//SetConsoleOutputCP(CP_UTF8);
|
|
|
|
converters_init();
|
|
|
|
udb_init();
|
|
|
|
//udb_dump();
|
|
|
|
#if 0
|
|
detect_bad_urls();
|
|
return 0;
|
|
#endif
|
|
|
|
#if 0
|
|
std::vector<string_vec> rows;
|
|
std::string title;
|
|
string_vec col_titles;
|
|
load_column_text("ufoevid13.txt", rows, title, col_titles);
|
|
return 0;
|
|
#endif
|
|
|
|
bool status = false, utf8_flag = false;
|
|
|
|
unordered_string_set unique_urls;
|
|
|
|
uprintf("Convert Hatch UDB:\n");
|
|
if (!udb_convert())
|
|
panic("udb_convert() failed!");
|
|
uprintf("Success\n");
|
|
|
|
#if 1
|
|
uprintf("Convert NICAP:\n");
|
|
if (!convert_nicap(unique_urls))
|
|
panic("convert_nicap() failed!");
|
|
uprintf("Success\n");
|
|
|
|
uprintf("Convert Johnson:\n");
|
|
if (!convert_johnson())
|
|
panic("convert_johnson() failed!");
|
|
uprintf("Success\n");
|
|
|
|
uprintf("Convert Eberhart:\n");
|
|
if (!convert_eberhart(unique_urls))
|
|
panic("convert_eberthart() failed!");
|
|
uprintf("Success\n");
|
|
|
|
uprintf("Convert Trace:\n");
|
|
if (!convert_magnonia("trace.txt", "trace.json", "Trace", " [Trace Cases](https://www.thenightskyii.org/trace.html)"))
|
|
panic("convert_magnonia() failed!");
|
|
uprintf("Success\n");
|
|
|
|
uprintf("Convert Magnonia:\n");
|
|
if (!convert_magnonia("magnonia.txt", "magnonia.json"))
|
|
panic("convert_magnonia() failed!");
|
|
uprintf("Success\n");
|
|
|
|
uprintf("Convert Hall:\n");
|
|
if (!convert_hall())
|
|
panic("convert_hall() failed!");
|
|
uprintf("Success\n");
|
|
|
|
uprintf("Convert Bluebook Unknowns:\n");
|
|
if (!convert_bluebook_unknowns())
|
|
panic("convert_bluebook_unknowns failed!");
|
|
uprintf("Success\n");
|
|
#endif
|
|
|
|
uprintf("Total unique URL's: %u\n", (uint32_t)unique_urls.size());
|
|
|
|
string_vec urls;
|
|
for (const auto& s : unique_urls)
|
|
urls.push_back(s);
|
|
std::sort(urls.begin(), urls.end());
|
|
write_text_file("unique_urls.txt", urls, false);
|
|
uprintf("Wrote unique_urls.txt\n");
|
|
|
|
ufo_timeline timeline;
|
|
|
|
status = timeline.load_json("maj2.json", utf8_flag, "Maj2", true);
|
|
if (!status)
|
|
panic("Failed loading maj2.json");
|
|
for (uint32_t i = 0; i < timeline.size(); i++)
|
|
timeline[i].m_source_id = string_format("%s_%u", timeline[i].m_source.c_str(), i);
|
|
|
|
status = timeline.load_json("hatch_udb.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading hatch_udb.json");
|
|
|
|
status = timeline.load_json("nicap_db.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading nicap_db.json");
|
|
|
|
status = timeline.load_json("trace.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading trace.json");
|
|
|
|
status = timeline.load_json("magnonia.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading magnolia.json");
|
|
|
|
status = timeline.load_json("bb_unknowns.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading bb_unknowns.json");
|
|
|
|
status = timeline.load_json("ufo_evidence_hall.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading ufo_evidence_hall.json");
|
|
|
|
status = timeline.load_json("eberhart.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading eberhart.json");
|
|
|
|
status = timeline.load_json("johnson.json", utf8_flag, nullptr, false);
|
|
if (!status)
|
|
panic("Failed loading johnson.json");
|
|
|
|
for (uint32_t i = 0; i < timeline.size(); i++)
|
|
{
|
|
if (!timeline[i].m_begin_date.sanity_check())
|
|
panic("Date failed sanity check");
|
|
|
|
if (timeline[i].m_end_date.is_valid())
|
|
{
|
|
if (!timeline[i].m_end_date.sanity_check())
|
|
panic("Date failed sanity check");
|
|
}
|
|
|
|
if (timeline[i].m_alt_date.is_valid())
|
|
{
|
|
if (!timeline[i].m_alt_date.sanity_check())
|
|
panic("Date failed sanity check");
|
|
}
|
|
|
|
// roundtrip test
|
|
event_date test_date;
|
|
|
|
if (!test_date.parse(timeline[i].m_date_str.c_str(), false))
|
|
panic("Date failed sanity check");
|
|
|
|
if (test_date != timeline[i].m_begin_date)
|
|
panic("Date failed sanity check");
|
|
|
|
std::string test_str(timeline[i].m_begin_date.get_string());
|
|
if (test_str != timeline[i].m_date_str)
|
|
{
|
|
fprintf(stderr, "Date failed roundtrip: %s %s %s\n", timeline[i].m_source_id.c_str(), timeline[i].m_date_str.c_str(), test_str.c_str());
|
|
panic("Date failed sanity check");
|
|
}
|
|
|
|
if (timeline[i].m_end_date.is_valid())
|
|
{
|
|
if (!test_date.parse(timeline[i].m_end_date_str.c_str(), false))
|
|
panic("Date failed sanity check");
|
|
|
|
if (test_date != timeline[i].m_end_date)
|
|
panic("Date failed sanity check");
|
|
|
|
std::string test_str2(timeline[i].m_end_date.get_string());
|
|
if (test_str2 != timeline[i].m_end_date_str)
|
|
{
|
|
fprintf(stderr, "Date failed roundtrip: %s %s %s\n", timeline[i].m_source_id.c_str(), timeline[i].m_end_date_str.c_str(), test_str2.c_str());
|
|
panic("End date failed sanity check");
|
|
}
|
|
}
|
|
else if (timeline[i].m_end_date_str.size())
|
|
panic("Date failed sanity check");
|
|
|
|
if (timeline[i].m_alt_date.is_valid())
|
|
{
|
|
if (!test_date.parse(timeline[i].m_alt_date_str.c_str(), false))
|
|
panic("Date failed sanity check");
|
|
|
|
if (test_date != timeline[i].m_alt_date)
|
|
panic("Date failed sanity check");
|
|
|
|
std::string test_str3(timeline[i].m_alt_date.get_string());
|
|
if (test_str3 != timeline[i].m_alt_date_str)
|
|
{
|
|
fprintf(stderr, "Date failed roundtrip: %s %s %s\n", timeline[i].m_source_id.c_str(), timeline[i].m_alt_date_str.c_str(), test_str3.c_str());
|
|
panic("Alt date failed sanity check");
|
|
}
|
|
}
|
|
else if (timeline[i].m_alt_date_str.size())
|
|
panic("Date failed sanity check");
|
|
|
|
}
|
|
|
|
uprintf("Load success, %zu total events\n", timeline.get_events().size());
|
|
|
|
timeline.sort();
|
|
|
|
// Write majestic.json, then load it and verify that it saved and loaded correctly.
|
|
{
|
|
timeline.set_name("Majestic Timeline");
|
|
timeline.write_file("majestic.json", true);
|
|
|
|
ufo_timeline timeline_comp;
|
|
bool utf8_flag_comp;
|
|
if (!timeline_comp.load_json("majestic.json", utf8_flag_comp, nullptr, false))
|
|
panic("Failed loading majestic.json");
|
|
|
|
if (timeline.get_events().size() != timeline_comp.get_events().size())
|
|
panic("Failed loading timeline events JSON");
|
|
|
|
for (size_t i = 0; i < timeline.get_events().size(); i++)
|
|
if (timeline[i] != timeline_comp[i])
|
|
panic("Failed comparing majestic.json");
|
|
}
|
|
|
|
timeline.write_markdown("timeline.md");
|
|
|
|
uprintf("Processing successful\n");
|
|
|
|
return EXIT_SUCCESS;
|
|
}
|