Skip to content

Concurrency for issue generation #536

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jwakely wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Concurrency for issue generation #536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
941b333
Create cross-references between duplicate issues in second step
jwakely Dec 12, 2025
e5b42c2
Make most lwg::report_generator member functions const
jwakely Dec 13, 2025
5a10e59
Generate issue lists concurrently
jwakely Dec 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/issues.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ struct issue {
std::string submitter; // original submitter of the issue
chrono::year_month_day date; // date the issue was filed
chrono::year_month_day mod_date; // date the issue was last changed
std::set<std::string> duplicates; // sorted list of duplicate issues, stored as html anchor references.
std::map<int, std::string> duplicates; // duplicate issues, as number and formatted html anchor
std::string text; // text representing the issue
int priority = 99; // severity, 1 = critical, 4 = minor concern, 0 = trivial to resolve, 99 = not yet prioritised
std::string owner; // person identified as taking ownership of drafting/progressing the issue
Expand Down
89 changes: 62 additions & 27 deletions src/lists.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <chrono>
#include <cstdlib>
#include <fstream>
#include <future>
#include <iostream>
#include <iterator>
#include <map>
Expand All @@ -40,6 +41,7 @@
#include <string>
#include <tuple>
#include <unordered_map>
#include <utility>
#include <vector>

#include <filesystem>
Expand Down Expand Up @@ -218,8 +220,10 @@ namespace
}

// The title of the specified paper, formatted as an HTML title="..." attribute.
std::string paper_title_attr(std::string paper_number, lwg::metadata& meta) {
auto title = meta.paper_titles[paper_number];
std::string paper_title_attr(std::string const& paper_number, lwg::metadata const& meta) {
std::string title;
if (auto pos = meta.paper_titles.find(paper_number); pos != meta.paper_titles.end())
title = pos->second;
if (!title.empty())
{
title = lwg::replace_reserved_char(std::move(title), '&', "&amp;");
Expand All @@ -230,7 +234,7 @@ std::string paper_title_attr(std::string paper_number, lwg::metadata& meta) {
}

void format_issue_as_html(lwg::issue & is,
std::span<lwg::issue> issues,
std::span<const lwg::issue> issues,
lwg::metadata & meta) {

auto& section_db = meta.section_db;
Expand Down Expand Up @@ -302,10 +306,10 @@ void format_issue_as_html(lwg::issue & is,
// note <p><i>[NOTE CONTENTS]</i></p>
// !-- comments are simply erased
//
// In addition, as duplicate issues are discovered, the duplicates are marked up
// in the supplied range [first_issue,last_issue). Similarly, if an unexpected
// (unknown) section is discovered, it will be inserted into the supplied
// section index, 'section_db'.
// In addition, as duplicate issues are discovered, the duplicates are recorded
// in the issue for later processing.
// Similarly, if an unexpected (unknown) section is discovered,
// it will be inserted into the supplied section index, 'section_db'.
//
// The behavior is undefined unless the issues in the supplied span are sorted by issue-number.
//
Expand Down Expand Up @@ -414,8 +418,7 @@ void format_issue_as_html(lwg::issue & is,
}

if (!tag_stack.empty() and tag_stack.back() == "duplicate") {
n->duplicates.insert(make_html_anchor(is));
is.duplicates.insert(make_html_anchor(*n));
is.duplicates[num] = make_html_anchor(*n);
r.clear();
}
else {
Expand Down Expand Up @@ -488,16 +491,22 @@ void format_issue_as_html(lwg::issue & is,

void prepare_issues(std::span<lwg::issue> issues, lwg::metadata & meta) {
// Initially sort the issues by issue number, so each issue can be correctly 'format'ted
std::ranges::sort(issues, {}, &lwg::issue::num);

// Then we format the issues, which should be the last time we need to touch the issues themselves
// We may turn this into a two-stage process, analysing duplicates and then applying the links
// This will allow us to better express constness when the issues are used purely for reference.
// Currently, the 'format' function takes a span of non-const-issues purely to
// mark up information related to duplicates, so processing duplicates in a separate pass may
// clarify the code.
std::ranges::sort(issues, {}, &lwg::issue::num);

// Then we format the issues, which should be the last time we need to touch the issues themselves.
// The full list of issues is passed so that <iref> elements can be resolved to an issue.
for (auto & i : issues) { format_issue_as_html(i, issues, meta); }

// Process the duplicates found while formatting the HTML.
// Ensure that each issue in i->duplicates has i in its own set of duplicates.
for (auto& i : issues) {
for (auto& dup : i.duplicates) {
auto& dupi = *std::ranges::lower_bound(issues, dup.first, {}, &lwg::issue::num);
if (auto& rev = dupi.duplicates[i.num]; rev.empty())
rev = make_html_anchor(i);
}
}

// Issues will be routinely re-sorted in later code, but contents should be fixed after formatting.
// This suggests we may want to be storing some kind of issue handle in the functions that keep
// re-sorting issues, and so minimize the churn on the larger objects.
Expand Down Expand Up @@ -716,6 +725,18 @@ void check_is_directory(fs::path const & directory) {
}
}

// Notes on performance (as of December 2025):
// Reading the XML files for each issues takes about 10% of the total run time.
// Converting the issue text to HTML takes about 10%.
// Generating the three main lists (active, defects, closed) takes about 20%.
// Generating the individual HTML pages for each issue takes about 35%.
//
// The cost of copying the vectors of issues and sorting them repeatedly is insignificant.
//
// Converting issues to HTML cannot be parallelized currently because it
// involves non-const accesses to the section_db, but it's not worth doing
// when it only takes 10% of the total time anyway.

int main(int argc, char* argv[]) {
try {
fs::path path;
Expand Down Expand Up @@ -804,20 +825,34 @@ int main(int argc, char* argv[]) {
: std::back_inserter(unresolved_issues);
std::copy_if(issues.begin(), issues.end(), ready_inserter, [](lwg::issue const & iss){ return lwg::is_ready(iss.stat); } );

using span = std::span<const lwg::issue>;
const auto launch = std::launch::async; // use deferred to serialize

// First generate the primary 3 standard issues lists
generator.make_active(issues, target_path, diff_report);
generator.make_defect(issues, target_path, diff_report);
generator.make_closed(issues, target_path, diff_report);

// unofficial documents
generator.make_tentative (issues, target_path);
generator.make_unresolved(issues, target_path);
generator.make_immediate (issues, target_path);
generator.make_ready (issues, target_path);
generator.make_editors_issues(issues, target_path);
generator.make_individual_issues(issues, target_path);
auto fut_active = std::async(launch, &lwg::report_generator::make_active,
std::cref(generator), span(issues), std::cref(target_path), std::cref(diff_report));
auto fut_defects = std::async(launch, &lwg::report_generator::make_defect,
std::cref(generator), span(issues), std::cref(target_path), std::cref(diff_report));
auto fut_closed = std::async(launch, &lwg::report_generator::make_closed,
std::cref(generator), span(issues), std::cref(target_path), std::cref(diff_report));

// unofficial documents
std::as_const(generator).make_tentative (issues, target_path);
std::as_const(generator).make_unresolved(issues, target_path);
std::as_const(generator).make_immediate (issues, target_path);
std::as_const(generator).make_ready (issues, target_path);
std::as_const(generator).make_editors_issues(issues, target_path);

// We need to join the concurrent tasks because make_individual_issues is non-const
// We could run the three make_sort_by_num calls before joining the futures,
// because those are const, except for re-sorting the issues span by issue number,
// but as they run first the issues are actually already sorted.
fut_active.wait();
fut_defects.wait();
fut_closed.wait();

generator.make_individual_issues(issues, target_path);

// Now we have a parsed and formatted set of issues, we can write the standard set of HTML documents
// Note that each of these functions is going to re-sort the 'issues' vector for its own purposes
Expand Down
42 changes: 20 additions & 22 deletions src/report_generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,16 +146,6 @@ auto to_string(major_section_key sn) -> std::string {
return out.str();
}

template<typename Container>
void print_list(std::ostream & out, Container const & source, char const * separator) {
char const * sep{""};
for (auto const & x : source) {
out << sep << x;
sep = separator;
}
}



void print_file_header(std::ostream& out, std::string const & title, std::string url_filename = {}, std::string desc = {}) {
out <<
Expand Down Expand Up @@ -301,7 +291,11 @@ R"(<table class="issues-index">

// Duplicates
out << "<td>";
print_list(out, i.duplicates, ", ");
char const* sep = "";
for (auto const& x : i.duplicates) {
out << sep << x.second;
sep = ", ";
}
out << "</td>\n"
<< "</tr>\n";
}
Expand Down Expand Up @@ -380,7 +374,11 @@ void print_issue(std::ostream & out, lwg::issue const & iss, lwg::section_map &
// duplicates
if (!iss.duplicates.empty()) {
out << "<p><b>Duplicate of:</b> ";
print_list(out, iss.duplicates, ", ");
char const* sep = "";
for (auto const& x : iss.duplicates) {
out << sep << x.second;
sep = ", ";
}
out << "</p>\n";
}

Expand Down Expand Up @@ -479,7 +477,7 @@ namespace lwg
// A precondition for calling any of these functions is that the list of issues is sorted in numerical order, by issue number.
// While nothing disastrous will happen if this precondition is violated, the published issues list will list items
// in the wrong order.
void report_generator::make_active(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) {
void report_generator::make_active(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) const {
assert(std::ranges::is_sorted(issues, {}, &issue::num));

fs::path filename{path / "lwg-active.html"};
Expand All @@ -498,7 +496,7 @@ void report_generator::make_active(std::span<const issue> issues, fs::path const
}


void report_generator::make_defect(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) {
void report_generator::make_defect(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) const {
assert(std::ranges::is_sorted(issues, {}, &issue::num));

fs::path filename{path / "lwg-defects.html"};
Expand All @@ -516,7 +514,7 @@ void report_generator::make_defect(std::span<const issue> issues, fs::path const
}


void report_generator::make_closed(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) {
void report_generator::make_closed(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) const {
assert(std::ranges::is_sorted(issues, {}, &issue::num));

fs::path filename{path / "lwg-closed.html"};
Expand All @@ -535,7 +533,7 @@ void report_generator::make_closed(std::span<const issue> issues, fs::path const


// Additional non-standard documents, useful for running LWG meetings
void report_generator::make_tentative(std::span<const issue> issues, fs::path const & path) {
void report_generator::make_tentative(std::span<const issue> issues, fs::path const & path) const {
// publish a document listing all tentative issues that may be acted on during a meeting.
assert(std::ranges::is_sorted(issues, {}, &issue::num));

Expand All @@ -555,7 +553,7 @@ void report_generator::make_tentative(std::span<const issue> issues, fs::path co
}


void report_generator::make_unresolved(std::span<const issue> issues, fs::path const & path) {
void report_generator::make_unresolved(std::span<const issue> issues, fs::path const & path) const {
// publish a document listing all non-tentative, non-ready issues that must be reviewed during a meeting.
assert(std::ranges::is_sorted(issues, {}, &issue::num));

Expand All @@ -574,7 +572,7 @@ void report_generator::make_unresolved(std::span<const issue> issues, fs::path c
print_file_trailer(out);
}

void report_generator::make_immediate(std::span<const issue> issues, fs::path const & path) {
void report_generator::make_immediate(std::span<const issue> issues, fs::path const & path) const {
// publish a document listing all non-tentative, non-ready issues that must be reviewed during a meeting.
assert(std::ranges::is_sorted(issues, {}, &issue::num));

Expand Down Expand Up @@ -609,7 +607,7 @@ out << R"(<h1>C++ Standard Library Issues Resolved Directly In [INSERT CURRENT M
print_file_trailer(out);
}

void report_generator::make_ready(std::span<const issue> issues, fs::path const & path) {
void report_generator::make_ready(std::span<const issue> issues, fs::path const & path) const {
// publish a document listing all ready issues for a formal vote
assert(std::ranges::is_sorted(issues, {}, &issue::num));

Expand Down Expand Up @@ -644,7 +642,7 @@ out << R"(<h1>C++ Standard Library Issues to be moved in [INSERT CURRENT MEETING
print_file_trailer(out);
}

void report_generator::make_editors_issues(std::span<const issue> issues, fs::path const & path) {
void report_generator::make_editors_issues(std::span<const issue> issues, fs::path const & path) const {
// publish a single document listing all 'Voting' and 'Immediate' resolutions (only).
assert(std::ranges::is_sorted(issues, {}, &issue::num));

Expand All @@ -659,7 +657,7 @@ void report_generator::make_editors_issues(std::span<const issue> issues, fs::pa
print_file_trailer(out);
}

void report_generator::make_sort_by_num(std::span<issue> issues, fs::path const & filename) {
void report_generator::make_sort_by_num(std::span<issue> issues, fs::path const & filename) const {
std::ranges::sort(issues, {}, &issue::num);

std::ofstream out{filename};
Expand Down Expand Up @@ -749,7 +747,7 @@ sorted by priority.</p>
print_file_trailer(out);
}

void report_generator::make_sort_by_status_impl(std::span<issue> issues, fs::path const & filename, std::string title) {
void report_generator::make_sort_by_status_impl(std::span<issue> issues, fs::path const & filename, std::string title) const {
std::ofstream out{filename};
if (!out)
throw std::runtime_error{"Failed to open " + filename.string()};
Expand Down
20 changes: 10 additions & 10 deletions src/report_generator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,27 +27,27 @@ struct report_generator {
// A precondition for calling any of these functions is that the list of issues is sorted in numerical order, by issue number.
// While nothing disastrous will happen if this precondition is violated, the published issues list will list items
// in the wrong order.
void make_active(std::span<const issue> issues, fs::path const & path, std::string const & diff_report);
void make_active(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) const;

void make_defect(std::span<const issue> issues, fs::path const & path, std::string const & diff_report);
void make_defect(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) const;

void make_closed(std::span<const issue> issues, fs::path const & path, std::string const & diff_report);
void make_closed(std::span<const issue> issues, fs::path const & path, std::string const & diff_report) const;

// Additional non-standard documents, useful for running LWG meetings
void make_tentative(std::span<const issue> issues, fs::path const & path);
void make_tentative(std::span<const issue> issues, fs::path const & path) const;
// publish a document listing all tentative issues that may be acted on during a meeting.


void make_unresolved(std::span<const issue> issues, fs::path const & path);
void make_unresolved(std::span<const issue> issues, fs::path const & path) const;
// publish a document listing all non-tentative, non-ready issues that must be reviewed during a meeting.

void make_immediate(std::span<const issue> issues, fs::path const & path);
void make_immediate(std::span<const issue> issues, fs::path const & path) const;
// publish a document listing all non-tentative, non-ready issues that must be reviewed during a meeting.

void make_ready(std::span<const issue> issues, fs::path const & path);
void make_ready(std::span<const issue> issues, fs::path const & path) const;
// publish a document listing all ready issues for a formal vote

void make_sort_by_num(std::span<issue> issues, fs::path const & filename);
void make_sort_by_num(std::span<issue> issues, fs::path const & filename) const;

void make_sort_by_priority(std::span<issue> issues, fs::path const & filename);

Expand All @@ -57,12 +57,12 @@ struct report_generator {

void make_sort_by_section(std::span<issue> issues, fs::path const & filename, bool active_only = false);

void make_editors_issues(std::span<const issue> issues, fs::path const & path);
void make_editors_issues(std::span<const issue> issues, fs::path const & path) const;

void make_individual_issues(std::span<const issue> issues, fs::path const & path);

private:
void make_sort_by_status_impl(std::span<issue> issues, fs::path const & filename, std::string title);
void make_sort_by_status_impl(std::span<issue> issues, fs::path const & filename, std::string title) const;

mailing_info const & lwg_issues_xml;
section_map & section_db;
Expand Down