Difference between revisions of "User:Space Mission"

//!
//! \abstract This program downloads the "Language support library macros"
//! [page](https://eel.is/c++draft/version.syn), parses it, and then generates the
//! wiki-source for the [page](https://en.cppreference.com/w/cpp/utility/feature_test).
//!
//! \usage: just compile & run, the wiki-page will be sent to terminal.
//!
//! \dependencies: the `curl` program, C++20.
//!
//! \author: (c) 2021. Space Mission. For cppreference.com internal usage.
//! \license: [CC-BY-SA](https://en.cppreference.com/w/Cppreference:Copyright/CC-BY-SA).
//!
//! \news:
//! 2021-06-12 : initial release (used an array that contains the plain text copied from
//!            : the eel page).
//! 2021-12-19 : added options and means to generate the whole wiki-page instead of only
//!            : the wiki-table. Note that this requires updates if the original
//!            : [page](cppreference.com/w/cpp/utility/feature_test) was changed.
//! 2021-12-20 : (optimizations) e.g. the output is generated w/o a temporary vector.
//! 2021-12-21 : added "curl" downloading of the source html file from
//!            : [the site](https://eel.is/c++draft/version.syn).
//! 2021-12-23 : added data extractor from the html using std::regex library.
//! 2021-12-27 : added more test.
//! 2022-01-29 : fixed SourceDownloader::load() always returned `false`; nonetheless,
//!            : everything worked due to try_local_file_if_download_failed == true.
//!
//! TODO: add "freestanding" support, see P2198R4 (Freestanding Feature-Test Macros)

#include <algorithm>
#include <cassert>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <regex>
#include <stdexcept>
#include <sstream>
#include <string>
#include <string_view>

using namespace std::literals;

/**
 * @brief      global options
 */
struct option {
    //
    //! A command to download the source html page. This string will be appended with the
    //! target file and then executed by std::system( command / tmp-dir / file-name );
    //!
    static inline constexpr auto command{
        "curl https://eel.is/c++draft/version.syn --silent -o "sv};
    //
    //! If set to `true` - generate the whole target page:
    //! [page](cppreference.com/w/cpp/utility/feature_test).
    //! Otherwise (`false`) generate the table only.
    static inline constexpr bool generate_the_whole_wiki_page {true};
    //
    //! Do not fail if the source html file was not loaded (due to Internet problems) in
    //! this session but, still exists locally (left from a previous session).
    //! Useful mostly for debugging.
    static inline constexpr bool try_local_file_if_download_failed {false};
    //
    //! Remove downloaded source html file (only if it was downloaded in this session).
    static inline constexpr bool remove_downloaded_file_at_exit {false};
    //
    //! Print additional info, such as success/failure.
    static inline constexpr bool verbose {false};
    //
    //! Test mode only.
    static inline constexpr bool enable_tests {false};
};

//!
//! \brief A downloader: gets the source file from the Internet.
//!
class SourceDownloader final {
    std::string downloaded_file_name_;

  public:
    SourceDownloader() = default;
    SourceDownloader(SourceDownloader const&) = default;
    SourceDownloader& operator=(SourceDownloader const&) = default;
    ~SourceDownloader() { cleanup(); }

  public:
    [[nodiscard]] bool load();
    [[nodiscard]] const std::string& file_name() const noexcept {
        return downloaded_file_name_;
    }
    void cleanup();
};

/**
 * @brief      Loads source html file from the "https://eel.is/c++draft/".
 * @return     return true if the file was downloaded successfully.
 */
inline bool SourceDownloader::load() {
    constexpr auto html {"eel_cpp_header_version.html"sv};

    downloaded_file_name_ = std::filesystem::temp_directory_path() / html;

    const std::string command {option::command.data() + downloaded_file_name_};

    if constexpr (option::verbose) {
        std::cout << "Downloading with: [" << command << "]\n";
        std::cout << "Destination file: " << downloaded_file_name_ << '\n';
    }

    if (const int ret_code {std::system(command.data())}; ret_code != 0) {
        if constexpr (option::verbose) {
            std::cout << "Can't download the file: error #" << ret_code << '\n';
            // TODO: maybe decipher the error return code.
        }
        return false;
    }
    if constexpr (option::verbose)
        std::cout << "OK. The file was downloaded successfully.\n";

    return true;
}

/**
 * @brief    Reset file and (conditionally) remove downloaded html source file.
 */
inline void SourceDownloader::cleanup() {
    if constexpr (option::remove_downloaded_file_at_exit) {
        if (!downloaded_file_name_.empty()) {
            std::filesystem::remove(downloaded_file_name_);
            downloaded_file_name_.clear();
        }
    }
}

/**
 * @brief class FeatureTestTable.
 *  Extracts data from a given string, that must be a part of
 *  [table entry](https://eel.is/c++draft/version.syn) html page
 *  and generates the the page [or (conditionally) only the table]:
 *  [page](https://en.cppreference.com/w/cpp/utility/feature_test)
 */
class FeatureTestTable {
  public:
    FeatureTestTable() = default;

  public:
    [[nodiscard]] std::string generate_entry(std::string_view source);

  private:
    [[nodiscard]] bool parse_line(std::string_view source);

  public:
    static bool generate();

  public:
    [[nodiscard]] static bool self_test();

  private:
    [[nodiscard]] static std::string_view page_head();
    [[nodiscard]] static std::string_view page_tail();
    [[nodiscard]] static std::string_view table_head();
    [[nodiscard]] static std::string_view table_tail();

  private:
    using string_view_vector = std::vector<std::string_view>;

  private:
    string_view_vector headers_;  // e.g. {"vector", "type_traits"}
    std::string_view macro_;      // e.g. "__cpp_lib_any"
    std::string_view date_;       // e.g. "201606L"
};

/**
 * @brief      Parses the source line and sets internal values for macro_, date_, headers_
 * @param[in]  source  - The source html line
 * @exception  std::logic_error - throws if the source html has a wrong format.
 * @return     true - if line was parsed successfully.
 */
inline bool FeatureTestTable::parse_line(std::string_view source) {
    headers_.clear();
    macro_ = date_ = "";

    const char* suffix;
    std::cmatch m;
    constexpr std::regex::flag_type flags{std::regex_constants::ECMAScript |
                                          std::regex_constants::optimize};

    /* Obtain macro (e.g. "__cpp_lib_byte") and date (e.g. "201603L")
     * HTML example:
    <span class='preprocessordirective'>#define</span> <span id='lib:__cpp_lib_byte'>...
    ...<span class='literal'>202011L</span>...
     */
    static const std::regex re_get_macro_and_date{
        "'lib:(__cpp_lib_[_a-z0-9]{3,50})'.*'literal'>(20[1-4][0-9]{3}L)",
        flags
    };
    if (!std::regex_search(source.data(), m, re_get_macro_and_date) or m.size() != 3)
        return false;
    macro_ = std::string_view(m[1].first, m[1].second - m[1].first);
    date_ = std::string_view(m[2].first, m[2].second - m[2].first);
    suffix = m.suffix().first;  // contains the tail with at least one header

    if (macro_.length() < "__cpp_lib_xxx"sv.length() or
        date_.length() != (/*a sample:*/ "202002L"sv).length())
        return false;

    /* Obtain header(s) in the cycle.
     * HTML example:
    ...<span id='headerref:<string_view>___'>...
     */
    static const std::regex re_find_header{
        R"regex((?:headerref:<)([_a-z\d]+(?:\.h)?)>)regex", flags};
    for (; std::regex_search(suffix, m, re_find_header) and m.size() == 2;
         suffix = m.suffix().first)  // contains the tail with zero or more headers
        headers_.emplace_back(m[1].first, m[1].second - m[1].first);

    return not headers_.empty();
}

/**
 * @brief      Performs self test and returns.
 *
 * @return     `true` if self-test passed.
 */
inline bool FeatureTestTable::self_test() {
    FeatureTestTable tab;

    constexpr auto fake_html =
    "<span class='preprocessordirective'>#define</span> <span id='lib:__cpp_lib_byte'>..."
    "...<span class='literal'>202011L</span>..."
    "...<span id='headerref:<string>___'>..."
    "...<span id='headerref:<string_view>___'>..."sv;

    return tab.parse_line(fake_html) and tab.macro_ == "__cpp_lib_byte" and
           tab.date_ == "202011L" and tab.headers_.size() == 2 and
           tab.headers_[0] == "string" and tab.headers_[1] == "string_view";
}


/**
 * @brief   cppreference page and table on the page related heads and tails.
 */
inline std::string_view FeatureTestTable::page_head() {
        if constexpr (option::generate_the_whole_wiki_page)
            return
        "{{title|Library feature-test macros {{mark c++20}}}}\n"
        "{{cpp/utility/navbar}}\n"
        "\n"
        "Each of following macros is defined if the header {{header|version}} or one of "
        "the corresponding headers specified in the table is included.\n"sv;
        else
            return ""sv;
        }

inline std::string_view FeatureTestTable::page_tail() {
        if constexpr (option::generate_the_whole_wiki_page)
            return
        "\n"
        "===See also===\n"
        "{{dsc begin}}\n"
        "{{dsc | [[cpp/feature_test|'''Feature testing''']] {{mark c++20}} | A set of "
        "preprocessor macros to test the corresponding to C++ language and library "
        "features }}\n"
        "{{dsc end}}\n"
        "\n"
        "{{langlinks|es|ja|ru|zh}}\n"sv;
        else
            return ""sv;
    }

inline std::string_view FeatureTestTable::table_head() {
        return
R"--(
{| class="wikitable sortable"
|-
! Macro name
! Value
! Header
)--"sv;
    }

inline std::string_view FeatureTestTable::table_tail() { return "|}"sv; }

/**
 * @brief      Generates cppreference table entry.
 * @param      source - source string to parse.
 * @return     non-empty table entry string, if success. An empty string otherwise.
 */
inline std::string FeatureTestTable::generate_entry(std::string_view source) {
    if (!parse_line(source))
        return {};

    /* cppreference table entry sample:
    |-
    |  {{tt|__cpp_lib_byte}}
    | 201603L
    | {{header|atomic}} {{header|filesystem}} ...
    */
    std::ostringstream str("", std::ios_base::ate);
    str << "|-\n"
           "| {{tt|" << macro_ << "}}\n"
           "| " << date_ << "\n"
           "|";
    for (auto const& header : headers_) {
        str << " {{header|" << header << "}}";
    }
    return str.str();
}

// TODO: ? implement output_iterator interface for SourceFileReader:
// begin(), end() [i.e. sentinel] operator*, and operator++.
class SourceFileReader {
public:
    SourceFileReader(SourceFileReader const&) = delete;
    SourceFileReader& operator=(SourceFileReader const&) = delete;

    explicit SourceFileReader(std::string_view const file_name);

    [[nodiscard]] bool is_open() const { return ok_; }
    [[nodiscard]] bool goto_next_line();
    [[nodiscard]] std::string_view get_line() const { return cur_; }

    operator std::ifstream& () noexcept { return file_; }

  private:
    std::ifstream file_;
    std::string cur_;
    std::string next_;
    bool ok_{false};
};

/**
 * @brief      Opens the source html file.
 *             Sets the success-flag that should be checked with is_open().
 * @param      file_name - the source html file name.
 */
inline SourceFileReader::SourceFileReader(std::string_view const file_name) {

    const auto file = std::filesystem::path{file_name};

    auto ec = std::error_code{};

    if (!std::filesystem::exists(file, ec)) {
        std::cerr << "ERROR: source html file not found: " << file << '\n';
        return;
    }

    constexpr auto min_file_size = std::uintmax_t{90'000}; // bytes
    if (auto size = std::uintmax_t{0};
        (size = std::filesystem::file_size(file, ec)) < min_file_size) {
        std::cerr << "ERROR: source html file: " << file << '\n'
                  << "  is too small, size = " << size << " bytes;\n"
                  << "  expected size >= " << min_file_size << " bytes\n";
        return;
    }

    file_.open(file_name.data());
    if (not(ok_ = file_.is_open())) {
        std::cerr << "ERROR: can't open the source html file: " << file << '\n';
        return;
    }
}

/**
 * @brief      Go to the next line, if it is available.
 * @exception  may throw std::logic_error in case of wrong file format
 * @return     true, if next line is available
 */
inline bool SourceFileReader::goto_next_line() {
    if (not ok_)
        return false;

    constexpr auto significant {"preprocessordirective"sv};

    if (cur_.empty()) { // This is a first run. Find the next valid line.
        while ((ok_ = !!std::getline(file_, cur_)) && cur_.find(significant) == ""sv.npos)
            ;
        if (not ok_) throw std::logic_error("Invalid file format.");
    } else {  // This is not the first run. Thus next_ contains the begin.
        cur_ = std::move(next_);
    }

    // Find the next begin (or EOF), appending the dependent lines.
    while ((ok_ = !!std::getline(file_, next_)) && next_.find(significant) == ""sv.npos) {
        cur_ += next_;
    }
    return true;
}

/**
 * @brief   Generates the whole cppreference page (or only the table) and prints it out.
 * @note    The format of the output is the wiki-media language. The printed result is to
 *          be applied to [page](https://en.cppreference.com/w/cpp/utility/feature_test).
 * @note    What will be printed (the page or mere the table) depends on the flag:
 *          option::generate_the_whole_wiki_page.
 */
inline bool FeatureTestTable::generate() {
    SourceDownloader html;
    if (not html.load()) {
        if constexpr (option::try_local_file_if_download_failed) {
            if constexpr (option::verbose)
                std::cerr << "Trying to use local source file: " << html.file_name()
                          << "\n\n";
        } else {
            return false;
        }
    }

    auto lines {SourceFileReader{html.file_name()}};
    if (not lines.is_open()) {
        return false;
    }

    std::cout << FeatureTestTable::page_head() << FeatureTestTable::table_head();

    auto entry_count {0U};

    try {
        for (FeatureTestTable table; lines.goto_next_line(); ) {
            const std::string line{ table.generate_entry(lines.get_line()) };
            if (not line.empty()) {
                std::cout << line << '\n';
                ++entry_count;
            }
        }
    } catch (std::logic_error const& ex) {
        if constexpr (option::verbose)
            std::cerr << "ERROR: " << ex.what() << '\n';
        return false;
    }

    std::cout << FeatureTestTable::table_tail() << '\n'
              << "<!-- Entries count: " << entry_count << " -->\n"
              << FeatureTestTable::page_tail();

    // Some sanity checks:
    constexpr auto min_entries {146u}; // as per 2021-12-24
    if (entry_count < min_entries) {
        std::cerr << "\nWARNING: Not enough entries! Expected at least " <<
            min_entries << '\n';
    }

    return true;
}

/**
 * @brief      Performs self test and returns.
 *
 * @return     `true` if self-tests passed.
 */
inline bool self_tests() {
    if constexpr (not option::enable_tests)
        return true;

    bool ok = FeatureTestTable::self_test();

    if (ok) {
        if constexpr (option::verbose)
            std::cerr << "OK. Tests passed.\n";
    } else {
        std::cerr << "TESTS FAILED.\n";
    }

    return ok;
}

int main() {
    return self_tests() and FeatureTestTable::generate() ? EXIT_SUCCESS : EXIT_FAILURE;
}