main.cpp

#include <algorithm>
#include <array>
#include <expected>
#include <filesystem>
#include <iterator>
#include <optional>
#include <string>
#include <utility>
#include <variant>

#include <CLI/CLI.hpp>
#include <boost/algorithm/string/split.hpp>
#include <boost/asio.hpp>
#include <boost/asio/basic_stream_file.hpp>
#include <boost/asio/buffer.hpp>
#include <boost/asio/error.hpp>
#include <boost/asio/file_base.hpp>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/read.hpp>
#include <boost/asio/ssl.hpp>
#include <boost/asio/stream_file.hpp>
#include <boost/asio/write.hpp>
#include <boost/beast.hpp>
#include <boost/beast/core/flat_buffer.hpp>
#include <boost/beast/core/tcp_stream.hpp>
#include <boost/beast/http/fields_fwd.hpp>
#include <boost/beast/http/message_fwd.hpp>
#include <boost/beast/http/string_body_fwd.hpp>
#include <boost/beast/http/verb.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/date_time.hpp>
#include <boost/process.hpp>
#include <boost/process/v2/environment.hpp>
#include <boost/process/v2/start_dir.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/ptree_fwd.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include <boost/range/iterator_range_core.hpp>
#include <boost/stacktrace.hpp>
#include <boost/system/detail/error_code.hpp>
#include <boost/url.hpp>
#include <corral/Nursery.h>
#include <corral/Semaphore.h>
#include <corral/Task.h>
#include <corral/asio.h>
#include <corral/corral.h>
#include <corral/detail/asio.h>
#include <corral/run.h>
#include <corral/utility.h>
#include <corral/wait.h>
#include <fmt/base.h>
#include <fmt/format.h>
#include <fmt/os.h>
#include <fmt/ostream.h>
#include <fmt/printf.h>
#include <fmt/std.h>
#include <magic_enum/magic_enum.hpp>
#include <quill/core/LogLevel.h>
#include <quill/std/Chrono.h>
#include <quill/std/FilesystemPath.h>
#include <quill/std/Vector.h>
#include <re2/re2.h>
#include <re2/set.h>
#include <reflex/pcre2matcher.h>

#include "gyou/apply_change.hpp"
#include "gyou/common_ctx_create.hpp"
#include "gyou/file_to_string.hpp"
#include "gyou/get_what_to_change.hpp"
#include "gyou/git_fetch.hpp"
#include "gyou/git_worktree.hpp"
#include "gyou/parsing_groupsci.hpp"
#include "gyou/string_to_file.hpp"
#include "gyou/structs/change_related/commit_specific.hpp"
#include "gyou/structs/common_ctx.hpp"
#include "gyou/structs/config.hpp"
#include "gyou/structs/consts.hpp"
#include "gyou/structs/result_of_parsing.hpp"
#include "gyou/structs/return_code.hpp"
#include "gyou/structs/service_enum.hpp"
#include "gyou/structs/service_regex.hpp"
#include "gyou/utils/array_utils.hpp"
#include "gyou/utils/boost_stacktrace_format.hpp"  // IWYU pragma: keep
#include "gyou/utils/omega_exception.hpp"
#include "gyou/utils/rusty_macros.hpp"
#include "gyou/utils/string_replace.hpp"
#include "gyou/utils/variants_utils.hpp"
#include "quill_usage/overwrite_log_macros.hpp"
#include "quill_usage/quill_setup.hpp"

namespace
{

    [[nodiscard]] corral::Task<gyou::ReturnCode> chief_logic(
        boost::asio::io_context& ioc, gyou::Config const& cfg,
        gyou::semaphores_array_type& semaphores,
        gyou::CommonContext& common_ctx)
    {
        boost::process::v2::filesystem::path git_exe_path
            = boost::process::environment::find_executable("git");

        std::optional<gyou::ReturnCode> return_code;

        gyou::PackagesToUpdate const changes = __extension__({
            gyou::PackagesToUpdate _changes;
            CORRAL_WITH_NURSERY(nursery)
            {
                nursery.start(
                    [&]() -> corral::Task<void>
                        {
                            _changes = co_await gyou::get_what_to_change(
                                ioc, cfg, semaphores, common_ctx);
                        });

                nursery.start(
                    [&]() -> corral::Task<void>
                        {
                            auto res = co_await gyou::git_fetch(ioc, cfg,
                                                                git_exe_path);
                            if (not res)
                                {
                                    LOG_ERROR(
                                        "`git fetch --all --prune` has failed. "
                                        "It's "
                                        "output:\n{}",
                                        std::move(res.error()));
                                    return_code
                                        = gyou::ReturnCode::FailedGitFetch;
                                    nursery.cancel();
                                    co_return;
                                }
                        });
                co_return corral::join;
            };
            std::move(_changes);
        });

        for (auto const& diff : changes.what_to_change)
            {
                match(
                    diff.data_for_how_to_change,
                    [&](gyou::EditCommit const& diff_commit)
                        {
                            LOG_INFO(
                                "Edit: path: '{}' old date: '{}' new date: "
                                "'{}' "
                                "old commit: '{}' new commit: '{}'",
                                diff.path_to_ebuild, diff_commit.old_ver.date,
                                diff_commit.new_ver.date,
                                diff_commit.old_ver.commit,
                                diff_commit.new_ver.commit);
                        },
                    [&](gyou::EditVerOrTag const& diff_ver_tag)
                        {
                            LOG_INFO(
                                "Edit: path: '{}' old ver: '{}' new ver: '{}'",
                                diff.path_to_ebuild, diff_ver_tag.old_ver,
                                diff_ver_tag.new_ver);
                        });
            }

        std::filesystem::path const path_to_grouping
            = cfg.path_to_repo / "groups-ci.json";

        std::string const str_grouping = __extension__({
            auto res = co_await gyou::file_to_string(ioc, path_to_grouping);
            if (!res)
                {
                    LOG_ERROR(
                        "Got error during getting the groups-ci.json file : "
                        "'{}'",
                        res.error().message());
                    co_return gyou::ReturnCode::FailedReadingGroupCiFile;
                };
            std::move(res.value());
        });

        gyou::GroupsCollection const groups = __extension__({
            auto res = gyou::parse_groups(str_grouping);
            if (!res)
                {
                    LOG_ERROR(
                        "Got error during parsing the groups-ci.json file : "
                        "'{}'",
                        std::move(res.error()));
                    co_return gyou::ReturnCode::FailedParsingGroupCiFile;
                }
            std::move(res.value());
        });

        std::unordered_multimap<size_t, size_t> const group_to_change
            = __extension__({
                  std::unordered_multimap<size_t, size_t> group_to_change_{};
                  group_to_change_.reserve(groups.amount_of_groups);

                  for (size_t change_index = 0;
                       change_index < std::size(changes.what_to_change);
                       ++change_index)
                      {
                          auto const& diff
                              = changes.what_to_change.at(change_index);
                          std::string const pv_to_look
                              = diff.path_to_ebuild.parent_path()
                                    .parent_path()
                                    .filename()
                                    .string()
                                + "/"
                                + diff.path_to_ebuild.parent_path()
                                      .filename()
                                      .string();
                          auto const it_str_to_group_index
                              = groups.groups.find(pv_to_look);
                          if (it_str_to_group_index != groups.groups.cend())
                              {
                                  LOG_TRACE_L1(
                                      "Found match to a group: pkg_name: '{}' "
                                      "group index: {}",
                                      pv_to_look,
                                      it_str_to_group_index->second);
                                  group_to_change_.emplace(
                                      it_str_to_group_index->second,
                                      change_index);
                              }
                          else
                              {
                                  LOG_TRACE_L1(
                                      "Have not found match to a group: "
                                      "pkg_name: '{}'",
                                      pv_to_look);
                                  group_to_change_.emplace(0, change_index);
                              }
                      }
                  std::move(group_to_change_);
              });

        std::filesystem::path const temp_folder_worktrees
            = cfg.path_to_tmp / "worktrees";
        std::error_code errc_mkdir_p_worktrees;
        std::filesystem::create_directories(temp_folder_worktrees,
                                            errc_mkdir_p_worktrees);
        if (errc_mkdir_p_worktrees)
            {
                LOG_ERROR("Failed to create temp directory for worktrees: {}",
                          errc_mkdir_p_worktrees.message());
                co_return gyou::ReturnCode::FailedCreateDirTmpWorktrees;
            };
        LOG_TRACE_L2("Created folder for worktrees.");

        std::filesystem::path const path_to_git = __extension__({
            auto path_to_git_
                = boost::process::environment::find_executable("git");
            if (path_to_git_.empty())
                {
                    LOG_ERROR(
                        "Failed to find 'git' executable in your environment.");
                    co_return gyou::ReturnCode::FailedToFindGit;
                }
            path_to_git_;
        });

        std::vector<std::string> branches;

        // this is actually pretty simple and stupid code, that looks ugly and I
        // do not know how to make it better.
        CORRAL_WITH_NURSERY(nursery)
        {
            auto worktree_create_change_and_pr
                = [&](corral::Nursery& nursery, std::string prefix,
                      auto range_grp_to_change_idx)
                {
                    std::filesystem::path const path_to_ebuild
                        = changes.what_to_change
                              .at(range_grp_to_change_idx.first->second)
                              .path_to_ebuild;
                    std::string const cat_pn = fmt::format(
                        "{}/{}",
                        path_to_ebuild.parent_path()
                            .parent_path()
                            .filename()
                            .string(),
                        path_to_ebuild.parent_path().filename().string());
                    std::filesystem::path const folder_path
                        = temp_folder_worktrees / (prefix + cat_pn);
                    std::string const branch_name
                        = fmt::format("ci_update/{}_{}", prefix, cat_pn);
                    branches.emplace_back(branch_name);

                    nursery.start(
                        [&, folder_path = folder_path,
                         branch_name = branch_name,
                         range_grp_to_change_idx
                         = range_grp_to_change_idx]() -> corral::Task<void>
                            {
                                {
                                    auto res = co_await git_create_worktree(
                                        ioc, cfg, path_to_git, folder_path,
                                        branch_name);
                                    if (not res)
                                        {
                                            LOG_ERROR(
                                                "Failed to create a git "
                                                "worktree: "
                                                "{}",
                                                std::move(res.error()));
                                            return_code = gyou::ReturnCode::
                                                FailedMakingGitToMakeWorktrees;
                                            nursery.cancel();
                                            co_return;
                                        }
                                }

                                for (auto it_grp_to_chg_idx
                                     = range_grp_to_change_idx.first;
                                     it_grp_to_chg_idx
                                     != range_grp_to_change_idx.second;
                                     ++it_grp_to_chg_idx)
                                    {
                                        nursery.start(
                                            [&, folder_path = folder_path,
                                             chg_idx
                                             = it_grp_to_chg_idx->second]()
                                                -> corral::Task<void>
                                                {
                                                    auto res = co_await gyou::
                                                        apply_change(
                                                            ioc, cfg,
                                                            folder_path,
                                                            changes
                                                                .what_to_change
                                                                .at(chg_idx));
                                                    if (not res)
                                                        {
                                                            LOG_ERROR(
                                                                "Failed to "
                                                                "apply "
                                                                "changes: "
                                                                "{}",
                                                                std::move(
                                                                    res.error()));
                                                            return_code = gyou::
                                                                ReturnCode::
                                                                    FailedApplyChange;
                                                            nursery.cancel();
                                                            co_return;
                                                        }
                                                });
                                    }
                            });
                };

            // logic for 0, i.e. no grouping
            auto range_grp_to_change_idx_no_grouping
                = group_to_change.equal_range(0);
            for (auto it_grp_to_chg_idx
                 = range_grp_to_change_idx_no_grouping.first;
                 it_grp_to_chg_idx
                 != range_grp_to_change_idx_no_grouping.second;
                 ++it_grp_to_chg_idx)
                {
                    worktree_create_change_and_pr(
                        nursery, "0_",
                        std::make_pair(it_grp_to_chg_idx,
                                       std::next(it_grp_to_chg_idx)));
                }

            // logic for groups

            for (size_t group_num = 1; group_num < groups.amount_of_groups + 1;
                 ++group_num)
                {
                    auto range_grp_to_change_idx
                        = group_to_change.equal_range(group_num);

                    worktree_create_change_and_pr(nursery,
                                                  fmt::format("{}_", group_num),
                                                  range_grp_to_change_idx);
                }

            co_return corral::join;
        };

        if (return_code.has_value())
            {
                co_return return_code.value();
            }

        if (changes.has_any_failed and changes.is_any_successful)
            {
                co_return gyou::ReturnCode::PartialSuccess;
            }
        else if (changes.has_any_failed and not changes.is_any_successful)
            {
                co_return gyou::ReturnCode::AllHaveFailed;
            }
        else if (not changes.is_any_successful)
            {
                co_return gyou::ReturnCode::NoEbuildFound;
            }
        else
            {
                co_return gyou::ReturnCode::Success;
                ;
            }
    }

    corral::Task<gyou::ReturnCode> actual_chief(boost::asio::io_context& ioc,
                                                gyou::Config const& cfg)
    {
        constexpr size_t amount_of_services
            = magic_enum::enum_count<gyou::Service>();

        std::array<corral::Semaphore, amount_of_services>
            semaphores_per_services
            = gyou::make_array_from_factory<amount_of_services>(
                [concurrency_per_service = cfg.concurrency_per_service]()
                    { return corral::Semaphore(concurrency_per_service); });

        gyou::CommonContext common_ctx = gyou::create_common_ctx();

        gyou::ReturnCode res = co_await chief_logic(
            ioc, cfg, semaphores_per_services, common_ctx);
        co_return res;
    }

}  // namespace

int main(int argc, char* argv[])
{
    gyou::Config cfg;

    // gyou::Config parsing
    {
        CLI::App app{
            "Post-processor for YouTube's RSS feed, so that you get "
            "summary of video inside the feed via sending an HTTP "
            "request to something like an Ollama instance."};

        // Temporary storage for CLI11 to map types it doesn't handle
        // natively without custom validators
        std::vector<std::string> headers_raw
            = {"Content-Type: application/json"};
        std::string log_level_str = "info";

        app.add_option("-H,--header", headers_raw,
                       "HTTP headers for request to an ?Ollama? instance.")
            ->take_all();

        app.add_option("-l,--log-file", cfg.log_file,
                       "Filepath to internal logs")
            ->default_val("./logs.log");

        app.add_option("-R,--repo-path", cfg.path_to_repo, "Filepath to repo")
            ->required();

        app.add_option("-t,--tmp-path", cfg.path_to_tmp,
                       "Filepath to portage's tmp")
            ->required();

        app.add_option("-G,--gentoo-repo-path", cfg.path_to_gentoo_repo,
                       "Filepath to gentoo's repo path")
            ->default_val("/var/db/repos/gentoo/");

        app.add_option("-P,--portage-bin-path", cfg.path_to_portage_bin,
                       "Filepath to portage's bin folder i.e. `git clone "
                       "https://github.com/Arniiiii/portage` and give the path "
                       "to `./bin/` folder")
            ->required();

        app.add_option("-Y,--portage-pym-path", cfg.path_to_portage_pym,
                       "Filepath to portage's PYM folder. I have no idea what "
                       "this is, but it can be any empty folder. ")
            ->required();

        app.add_option("--log-level", log_level_str,
                       "Log level: "
                       "tracel3,tracel2,tracel1,debug,info,notice,warning,"
                       "error,critical")
            ->default_val("info");
        app.add_option("--main-branch-name", cfg.main_branch_name,
                       "Main branch name of the repo ")
            ->default_val("master");

        app.add_option("-J,--jobs-requests", cfg.concurrency_per_service,
                       "Amount of concurrent request per service"
                       "by this application")
            ->check(CLI::PositiveNumber)
            ->default_val(gyou::DEFAULT_MAX_CONCURRENT_REQUESTS_PER_SERVICE);
        try
            {
                app.parse(argc, argv);

                auto const path_to_ebuild_sh
                    = cfg.path_to_portage_bin / "ebuild.sh";
                if (not std::filesystem::exists(path_to_ebuild_sh))
                    {
                        throw OmegaException<std::filesystem::path>(
                            "An expected executable does not exists.",
                            path_to_ebuild_sh);
                    }

                cfg.log_level = quill::loglevel_from_string(log_level_str);

                for (const auto& header_raw_str : headers_raw)
                    {
                        std::vector<std::string> parts;
                        boost::split(parts, header_raw_str,
                                     boost::is_any_of(":"));
                        if (parts.size() >= 2)
                            {
                                std::string key = boost::trim_copy(parts[0]);
                                std::string val = boost::trim_copy(parts[1]);
                                cfg.headers.insert(key, val);
                            }
                    }
            }
        catch (const CLI::ParseError& e)
            {
                return app.exit(e);
            }
        catch (OmegaException<std::filesystem::path>& e)
            {
                boost::stacktrace::basic_stacktrace<
                    std::allocator<boost::stacktrace::frame>>
                    trace
                    = boost::stacktrace::stacktrace::from_current_exception();
                std::string log = fmt::format(
                    R"(Oohh, look at you, who got an exception, my cutie lovely guy. 
This is an Omega exception. Most definitely an incorrect value was specified in command line args, specifically a file path to something. 

Here's .what():
{}

Here's data:
{}

Here's trace:
{}

Here's line where something failed:
{}

Here's an attempt to get backtrace of it via boost::stacktrace and libbacktrace:
{}
)",
                    e.what(), e.data(), e.stack(), e.where(),
                    boost::stacktrace::to_string(trace));

                fmt::print(std::cerr, "{}", log);
                return std::to_underlying(
                    gyou::ReturnCode::FailSpecifiedValueIsIncorrect);
            }
        catch (OmegaException<std::string>& e)
            {
                boost::stacktrace::basic_stacktrace<
                    std::allocator<boost::stacktrace::frame>>
                    trace
                    = boost::stacktrace::stacktrace::from_current_exception();
                std::string log = fmt::format(
                    R"(Oohh, look at you, who got an exception, my cutie lovely guy. 
This is an Omega exception. Most definitely an incorrect value was specified in command line args. 

Here's .what():
{}

Here's data:
{}

Here's trace:
{}

Here's line where something failed:
{}

Here's an attempt to get backtrace of it via boost::stacktrace and libbacktrace:
{}
)",
                    e.what(), e.data(), e.stack(), e.where(),
                    boost::stacktrace::to_string(trace));

                fmt::print(std::cerr, "{}", log);
                return std::to_underlying(
                    gyou::ReturnCode::FailSpecifiedValueIsIncorrect);
            }
        catch (std::exception& e)
            {
                boost::stacktrace::basic_stacktrace<
                    std::allocator<boost::stacktrace::frame>>
                    trace
                    = boost::stacktrace::stacktrace::from_current_exception();
                fmt::print(
                    std::cerr,
                    R"(Oohh, look at you, who got an exception, my cutie lovely guy. 
You know that standard exceptions sucks. 
Also, this exception is from the part of parsing command line args. Most definitely an incorrect value was specified in args.
                
Here's .what():
{}

Here's an attempt to get backtrace of it via boost::stacktrace and libbacktrace... Hope it works or kill youself debugging this shit:
{}
)",
                    e.what(), boost::stacktrace::to_string(trace));
                return std::to_underlying(
                    gyou::ReturnCode::FailDuringInitializationConfig);
            }
    }

    // Enabling logging
    try
        {
            setup_quill(cfg.log_file, cfg.log_level);
        }
    catch (std::exception& e)
        {
            boost::stacktrace::basic_stacktrace<
                std::allocator<boost::stacktrace::frame>>
                trace = boost::stacktrace::stacktrace::from_current_exception();
            fmt::print(
                std::cerr,
                R"(Oohh, look at you, who got an exception, my cutie lovely guy.  
You know that standard exceptions sucks. 
Also, this exception appeared during setting up logging.

Here's .what():
{}

Here's an attempt to get backtrace of it via boost::stacktrace and libbacktrace... Hope it works or kill youself debugging this shit:
{}
)",
                e.what(), boost::stacktrace::to_string(trace));
            return std::to_underlying(
                gyou::ReturnCode::FailInitializationLogger);
        }

    // Finally main.
    try
        {
            boost::asio::io_context ioc;
            LOG_DEBUG("Entering asynchronous main...");
            boost::asio::signal_set signals(ioc, SIGINT, SIGTERM);
            auto&& [main_opt, signals_opt] = corral::run(
                ioc, corral::anyOf(actual_chief(ioc, cfg),
                                   signals.async_wait(corral::asio_awaitable)));
            if (main_opt)
                {
                    return std::to_underlying(main_opt.value());
                }
            return std::to_underlying(
                gyou::ReturnCode::ReceivedCancellationSignal);
        }
    catch (OmegaException<std::filesystem::path>& e)
        {
            boost::stacktrace::basic_stacktrace<
                std::allocator<boost::stacktrace::frame>>
                trace = boost::stacktrace::stacktrace::from_current_exception();
            std::string log = fmt::format(
                R"(Oohh, look at you, who got an exception, my cutie lovely guy. 
This is an Omega exception. Maybe from checking whether a file exists.

Here's .what():
{}

Here's data: 
{}

Here's trace:
{}

Here's line where something failed:
{}

Here's an attempt to get backtrace of it via boost::stacktrace and libbacktrace: 
{}
)",
                e.what(), e.data(), e.stack(), e.where(),
                boost::stacktrace::to_string(trace));

            fmt::print(std::cerr, "{}", log);
            LOG_ERROR("{}", log);
            return std::to_underlying(gyou::ReturnCode::FailParsePromptResult);
        }
    catch (OmegaException<std::string>& e)
        {
            boost::stacktrace::basic_stacktrace<
                std::allocator<boost::stacktrace::frame>>
                trace = boost::stacktrace::stacktrace::from_current_exception();
            std::string log = fmt::format(
                R"(Oohh, look at you, who got an exception, my cutie lovely guy. 
This is a generic Omega exception.

Here's .what():
{}

Here's data:
{}

Here's trace:
{}

Here's line where something failed:
{}

Here's an attempt to get backtrace of it via boost::stacktrace and libbacktrace:
{}
)",
                e.what(), e.data(), e.stack(), e.where(),
                boost::stacktrace::to_string(trace));

            fmt::print(std::cerr, "{}", log);
            LOG_ERROR("{}", log);
            return std::to_underlying(gyou::ReturnCode::FailParsePromptResult);
        }
    catch (std::exception& e)
        {
            boost::stacktrace::basic_stacktrace<
                std::allocator<boost::stacktrace::frame>>
                trace = boost::stacktrace::stacktrace::from_current_exception();
            std::string log = fmt::format(
                R"(Oohh, look at you, who got an exception, my cutie lovely guy. 
You know that standard exceptions sucks. 

Here's .what():
{}

Here's an attempt to get backtrace of it via boost::stacktrace and libbacktrace... Hope it works or kill youself debugging this shit:
{}
)",
                e.what(), boost::stacktrace::to_string(trace));

            fmt::print(std::cerr, "{}", log);
            LOG_ERROR("{}", log);
            return std::to_underlying(gyou::ReturnCode::FailStandardException);
        }
}