Compare commits

..
Author SHA1 Message Date
Niels Lohmann 4b538d80f6 Merge branch 'develop' into ci/path-filters
Keep ci_test_documentation in repository_checks.yml and apply the
harden-runner 2.22.2 bump from #5962 there.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-30 15:35:39 +01:00
Niels Lohmann 7a4e6b7a5d Merge develop into ci/path-filters
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-28 22:09:53 +01:00
Niels Lohmann fd37b04813 Merge origin/develop into ci/path-filters
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-28 20:32:10 +01:00
Niels Lohmann bd442d5d07 Merge develop
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-28 07:11:43 +01:00
Niels Lohmann eae3c8217b Split the checks of check_path_filters.py into smaller functions
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-27 19:39:20 +01:00
Niels Lohmann 960c234cc6 Do not report the excluded files that check_path_filters.py lists itself
Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-27 18:58:36 +01:00
Niels Lohmann b6e24d19a9 Skip the compiler workflows for pull requests that change no build input
ubuntu.yml, macos.yml and windows.yml run about 155 jobs on every pull
request, including those that change only the documentation. They now
have a pull_request path filter: it starts with '**' and excludes the
documentation, Markdown files, GitHub metadata, other workflows and the
license files, so a new file runs them unless it is excluded on purpose.
Pushes to develop, master and release/* are not filtered.

The documentation jobs and the REUSE check move from ubuntu.yml to the
new repository_checks.yml, which has no path filter and also runs
tools/check_path_filters. That check keeps the three filters identical
and fails when a file the compiler jobs keep mentions an excluded file.
It found two: tools/check_build_options reads the Meson section of
package_managers.md, and the swiftpm job builds the Swift example from
the documentation; both are included in the filter.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>
2026-10-27 18:35:03 +01:00
13 changed files with 488 additions and 161 deletions

No files matched your search

+5
View File
@@ -154,6 +154,11 @@ and each page's title (H1) and declaration by
[`docs/docset/generate_docset.py`](https://github.com/nlohmann/json/blob/develop/docs/docset/generate_docset.py);
`make build` reports API pages that cannot be classified.
A pull request that changes only the documentation, Markdown files, GitHub metadata, or license files skips the
compiler workflows (Ubuntu, macOS, Windows); the documentation and license checks still run. See
[`tools/check_path_filters`](https://github.com/nlohmann/json/tree/develop/tools/check_path_filters) for the list of
skipped files and what to do when a compiler job starts reading one of them.
### Amalgamate the source code
The single-header files
+19
View File
@@ -7,6 +7,25 @@ on:
- master
- release/*
pull_request:
# Skip pull requests that change only files no job here reads. The list is checked by
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
paths:
- '**'
- '!docs/**'
- '!**.md'
- '!.github/*'
- '!.github/ISSUE_TEMPLATE/**'
- '!.github/external_ci/**'
- '!.github/workflows/**'
- '!.reuse/**'
- '!LICENSES/**'
- '!CITATION.cff'
- '!REUSE.toml'
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
- 'docs/mkdocs/docs/integration/package_managers.md'
# built by the swiftpm job (macos.yml)
- 'docs/mkdocs/docs/integration/swift/**'
- '.github/workflows/macos.yml'
workflow_dispatch:
concurrency:
+78
View File
@@ -0,0 +1,78 @@
name: Repository checks
# Checks that concern files the compiler workflows (ubuntu.yml, macos.yml,
# windows.yml) skip with their path filter: the documentation, the license
# information, and the path filter itself. This workflow has no path filter.
on:
push:
branches:
- develop
- master
- release/*
pull_request:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
cancel-in-progress: true
jobs:
check_path_filters:
runs-on: ubuntu-26.04
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Check the path filters of the compiler workflows
run: python3 tools/check_path_filters/check_path_filters.py
ci_reuse_compliance:
runs-on: ubuntu-26.04
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Get latest CMake and ninja
uses: lukka/get-cmake@a89e43e22a96abafcd7a62db652a65ecd33052c9 # v4.4.4
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
run: cmake --build build --target ci_reuse_compliance
ci_test_documentation:
runs-on: ubuntu-26.04
strategy:
matrix:
target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid]
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail
fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }}
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
if: matrix.target == 'ci_test_documentation_mermaid'
with:
node-version: 24
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
run: cmake --build build --target ${{ matrix.target }}
+20 -26
View File
@@ -7,6 +7,25 @@ on:
- master
- release/*
pull_request:
# Skip pull requests that change only files no job here reads. The list is checked by
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
paths:
- '**'
- '!docs/**'
- '!**.md'
- '!.github/*'
- '!.github/ISSUE_TEMPLATE/**'
- '!.github/external_ci/**'
- '!.github/workflows/**'
- '!.reuse/**'
- '!LICENSES/**'
- '!CITATION.cff'
- '!REUSE.toml'
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
- 'docs/mkdocs/docs/integration/package_managers.md'
# built by the swiftpm job (macos.yml)
- 'docs/mkdocs/docs/integration/swift/**'
- '.github/workflows/ubuntu.yml'
workflow_dispatch:
permissions:
@@ -109,7 +128,7 @@ jobs:
runs-on: ubuntu-26.04
strategy:
matrix:
target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_reuse_compliance, ci_test_valgrind]
target: [ci_test_single_header, ci_cppcheck, ci_cpplint, ci_reproducible_tests, ci_non_git_tests, ci_offline_testdata, ci_test_valgrind]
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
@@ -449,31 +468,6 @@ jobs:
- name: Build
run: cmake --build build
ci_test_documentation:
runs-on: ubuntu-26.04
strategy:
matrix:
target: [ci_test_examples, ci_test_build_documentation, ci_test_documentation_mermaid]
steps:
- name: Harden Runner
uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
# the git-revision-date-localized plugin needs the history; a shallow clone makes the strict build fail
fetch-depth: ${{ matrix.target == 'ci_test_build_documentation' && '0' || '1' }}
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
if: matrix.target == 'ci_test_documentation_mermaid'
with:
node-version: 24
- name: Run CMake
run: cmake -S . -B build -DJSON_CI=On
- name: Build
run: cmake --build build --target ${{ matrix.target }}
# Native Linux ARM64 runner with the latest GCC. The ubuntu-24.04-arm label is
# only available for public repositories.
ci_test_arm64:
+19
View File
@@ -7,6 +7,25 @@ on:
- master
- release/*
pull_request:
# Skip pull requests that change only files no job here reads. The list is checked by
# tools/check_path_filters/check_path_filters.py (repository_checks.yml); change both together.
paths:
- '**'
- '!docs/**'
- '!**.md'
- '!.github/*'
- '!.github/ISSUE_TEMPLATE/**'
- '!.github/external_ci/**'
- '!.github/workflows/**'
- '!.reuse/**'
- '!LICENSES/**'
- '!CITATION.cff'
- '!REUSE.toml'
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
- 'docs/mkdocs/docs/integration/package_managers.md'
# built by the swiftpm job (macos.yml)
- 'docs/mkdocs/docs/integration/swift/**'
- '.github/workflows/windows.yml'
workflow_dispatch:
permissions:
+18 -22
View File
@@ -465,17 +465,15 @@ class binary_reader
}
const std::size_t element_type_parse_position = chars_read;
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on: they are skipped unless
// error_handler_t::strict must reject ill-formed UTF-8 in them
const bool is_object = container_stack.back().is_object;
key.clear();
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
{
return false;
}
if (is_object && !sax->key(key))
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on
if (container_stack.back().is_object && !sax->key(key))
{
return false;
}
@@ -491,18 +489,14 @@ class binary_reader
@brief Parses a C-style string from the BSON input.
@param[in,out] result A reference to the string variable where the read
string is to be stored.
@param[in] store whether the characters are needed; if not (the names of
an array's elements), they are skipped and @a result may
be left unchanged, and they are not checked against
@ref error_handler
@return `true` if the \\x00-byte indicating the end of the string was
encountered before the EOF; false` indicates an unexpected EOF.
*/
bool get_bson_cstr(string_t& result, const bool store = true)
bool get_bson_cstr(string_t& result)
{
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
{
return !store || check_string_utf8(result, "key");
return check_string_utf8(result, "key");
}
auto out = std::back_inserter(result);
@@ -515,7 +509,7 @@ class binary_reader
}
if (current == 0x00)
{
return !store || check_string_utf8(result, "key");
return check_string_utf8(result, "key");
}
*out++ = static_cast<typename string_t::value_type>(current);
}
@@ -525,11 +519,10 @@ class binary_reader
@brief read a C-style string from contiguous input in one step
@param[in,out] result the string to append to
@param[in] store whether to append the characters to @a result
@return whether the string was read; if the input has no \\x00-byte, nothing
is read, and @ref get_bson_cstr reports the end of the input
*/
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
{
const std::size_t remaining = ia.bulk_remaining();
if (remaining == 0)
@@ -537,15 +530,18 @@ class binary_reader
return false;
}
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const std::size_t length = find_zero_byte(data, remaining);
// a plain loop rather than std::memchr: most keys are short (array
// indices are keys, too), and the call would cost more than it saves
std::size_t length = 0;
while (length < remaining && data[length] != 0x00)
{
++length;
}
if (length == remaining)
{
return false;
}
if (store)
{
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
}
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
// consume the string and its \x00-byte, as the byte-wise path does
ia.bulk_skip(length + 1);
chars_read += length + 1;
@@ -554,7 +550,7 @@ class binary_reader
}
/// input that is not contiguous: C-style strings are read byte by byte
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
{
return false;
}
@@ -90,34 +90,6 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
return n;
}
// return the index of the first 0x00 byte in [data, data+n), or n if there is
// none; scans 8 bytes at a time (BSON's C-style strings)
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
{
constexpr std::uint64_t ones = 0x0101010101010101ull;
constexpr std::uint64_t high = 0x8080808080808080ull;
std::size_t i = 0;
for (; i + 8 <= n; i += 8)
{
const std::uint64_t v = read_eight_bytes(data + i);
const std::uint64_t zero = (v - ones) & ~v & high;
if (zero != 0)
{
// the lowest flagged byte is the first zero byte (see
// find_string_special())
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
}
}
for (; i < n; ++i)
{
if (data[i] == 0x00)
{
return i;
}
}
return n;
}
// classify a byte as one the serializer must NOT copy verbatim when
// ensure_ascii is requested: the closing quote, an escape, a control character
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
+3
View File
@@ -766,6 +766,9 @@ class basic_json_view
template<typename, typename> friend class detail::view::view_side;
friend iterator;
/// whether the view is one of an editable document
static constexpr bool is_editable = Editable;
basic_json_view(const document_data* d, const node* n) noexcept
: m_doc(d), m_node(n)
{}
+18 -50
View File
@@ -12458,34 +12458,6 @@ JSON_HEDLEY_ALWAYS_INLINE std::size_t find_string_special(const unsigned char* d
return n;
}
// return the index of the first 0x00 byte in [data, data+n), or n if there is
// none; scans 8 bytes at a time (BSON's C-style strings)
JSON_HEDLEY_ALWAYS_INLINE std::size_t find_zero_byte(const unsigned char* data, std::size_t n) noexcept
{
constexpr std::uint64_t ones = 0x0101010101010101ull;
constexpr std::uint64_t high = 0x8080808080808080ull;
std::size_t i = 0;
for (; i + 8 <= n; i += 8)
{
const std::uint64_t v = read_eight_bytes(data + i);
const std::uint64_t zero = (v - ones) & ~v & high;
if (zero != 0)
{
// the lowest flagged byte is the first zero byte (see
// find_string_special())
return i + (static_cast<std::size_t>(count_trailing_zeros(zero)) / 8);
}
}
for (; i < n; ++i)
{
if (data[i] == 0x00)
{
return i;
}
}
return n;
}
// classify a byte as one the serializer must NOT copy verbatim when
// ensure_ascii is requested: the closing quote, an escape, a control character
// (< 0x20), DEL (0x7F), or any non-ASCII byte (>= 0x80). Everything else -
@@ -18269,17 +18241,15 @@ class binary_reader
}
const std::size_t element_type_parse_position = chars_read;
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on: they are skipped unless
// error_handler_t::strict must reject ill-formed UTF-8 in them
const bool is_object = container_stack.back().is_object;
key.clear();
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key, is_object || error_handler == error_handler_t::strict)))
if (JSON_HEDLEY_UNLIKELY(!get_bson_cstr(key)))
{
return false;
}
if (is_object && !sax->key(key))
// an array's elements are named "0", "1", ... in the wire format,
// and those names are not passed on
if (container_stack.back().is_object && !sax->key(key))
{
return false;
}
@@ -18295,18 +18265,14 @@ class binary_reader
@brief Parses a C-style string from the BSON input.
@param[in,out] result A reference to the string variable where the read
string is to be stored.
@param[in] store whether the characters are needed; if not (the names of
an array's elements), they are skipped and @a result may
be left unchanged, and they are not checked against
@ref error_handler
@return `true` if the \\x00-byte indicating the end of the string was
encountered before the EOF; false` indicates an unexpected EOF.
*/
bool get_bson_cstr(string_t& result, const bool store = true)
bool get_bson_cstr(string_t& result)
{
if (get_bson_cstr_bulk(result, store, std::integral_constant<bool, bulk_scan> {}))
if (get_bson_cstr_bulk(result, std::integral_constant<bool, bulk_scan> {}))
{
return !store || check_string_utf8(result, "key");
return check_string_utf8(result, "key");
}
auto out = std::back_inserter(result);
@@ -18319,7 +18285,7 @@ class binary_reader
}
if (current == 0x00)
{
return !store || check_string_utf8(result, "key");
return check_string_utf8(result, "key");
}
*out++ = static_cast<typename string_t::value_type>(current);
}
@@ -18329,11 +18295,10 @@ class binary_reader
@brief read a C-style string from contiguous input in one step
@param[in,out] result the string to append to
@param[in] store whether to append the characters to @a result
@return whether the string was read; if the input has no \\x00-byte, nothing
is read, and @ref get_bson_cstr reports the end of the input
*/
bool get_bson_cstr_bulk(string_t& result, const bool store, std::true_type /*bulk*/)
bool get_bson_cstr_bulk(string_t& result, std::true_type /*bulk*/)
{
const std::size_t remaining = ia.bulk_remaining();
if (remaining == 0)
@@ -18341,15 +18306,18 @@ class binary_reader
return false;
}
const auto* const data = reinterpret_cast<const unsigned char*>(ia.bulk_data());
const std::size_t length = find_zero_byte(data, remaining);
// a plain loop rather than std::memchr: most keys are short (array
// indices are keys, too), and the call would cost more than it saves
std::size_t length = 0;
while (length < remaining && data[length] != 0x00)
{
++length;
}
if (length == remaining)
{
return false;
}
if (store)
{
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
}
result.append(reinterpret_cast<const typename string_t::value_type*>(data), length);
// consume the string and its \x00-byte, as the byte-wise path does
ia.bulk_skip(length + 1);
chars_read += length + 1;
@@ -18358,7 +18326,7 @@ class binary_reader
}
/// input that is not contiguous: C-style strings are read byte by byte
bool get_bson_cstr_bulk(string_t& /*result*/, const bool /*store*/, std::false_type /*bulk*/) const noexcept
bool get_bson_cstr_bulk(string_t& /*result*/, std::false_type /*bulk*/) const noexcept
{
return false;
}
+3
View File
@@ -9807,6 +9807,9 @@ class basic_json_view
template<typename, typename> friend class detail::view::view_side;
friend iterator;
/// whether the view is one of an editable document
static constexpr bool is_editable = Editable;
basic_json_view(const document_data* d, const node* n) noexcept
: m_doc(d), m_node(n)
{}
+2 -35
View File
@@ -1164,13 +1164,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
// contiguous input reads a key up to its \x00-byte in one step, a stream
// reads it byte by byte; both must give the same value or error for the
// complete document and for every truncation of it
json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
// keys and array element names ("0" to "11") that end at every position of
// an 8-byte word, which contiguous input scans at a time
for (std::size_t length = 1; length <= 17; ++length)
{
j[std::string(length, static_cast<char>('a' + length))] = json::array({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, {11}});
}
const json j = {{"", true}, {"k", {1, 2, 3}}, {std::string(40, 'x'), {{"nested key", "value"}}}};
const std::vector<std::uint8_t> bson = json::to_bson(j);
CHECK(json::from_bson(bson) == j);
@@ -1190,7 +1184,7 @@ TEST_CASE("BSON keys from contiguous and stream input")
}
try
{
std::istringstream stream(std::string(reinterpret_cast<const char*>(input.data()), input.size()));
std::istringstream stream(std::string(input.begin(), input.end()));
from_stream = json::from_bson(stream).dump();
}
catch (const json::parse_error& e)
@@ -1200,33 +1194,6 @@ TEST_CASE("BSON keys from contiguous and stream input")
CHECK(from_vector == from_stream);
}
}
TEST_CASE("BSON array element names")
{
// an array's element names are not part of the value, so they are not
// passed on; error_handler_t::strict still rejects ill-formed UTF-8 in them
const std::vector<std::uint8_t> bson =
{
0x1D, 0x00, 0x00, 0x00, // size (little endian)
0x04, 'a', 0x00, // entry: array "a"
0x15, 0x00, 0x00, 0x00, // size (little endian)
0x08, 0xFF, 0x00, 0x01, // entry: boolean named "\xFF", true
0x08, '1', '2', '3', '4', '5', '6', '7', '8', '9', 0x00, 0x00, // entry: boolean named "123456789", false
0x00, // end of the array
0x00 // end of the document
};
const json expected = {{"a", {true, false}}};
CHECK(json::from_bson(bson) == expected);
CHECK(json::from_bson(bson, true, true, json::error_handler_t::replace) == expected);
CHECK(json::from_bson(bson, true, true, json::error_handler_t::ignore) == expected);
json _;
CHECK_THROWS_WITH_AS(_ = json::from_bson(bson, true, true, json::error_handler_t::strict), "[json.exception.parse_error.113] parse error at byte 14: syntax error while parsing BSON key: invalid string: ill-formed UTF-8 byte", json::parse_error&);
std::istringstream stream(std::string(reinterpret_cast<const char*>(bson.data()), bson.size()));
CHECK(json::from_bson(stream) == expected);
}
#endif
TEST_CASE("Negative size of binary value")
+34
View File
@@ -0,0 +1,34 @@
# check_path_filters
The compiler workflows [`ubuntu.yml`](../../.github/workflows/ubuntu.yml),
[`macos.yml`](../../.github/workflows/macos.yml), and [`windows.yml`](../../.github/workflows/windows.yml) skip pull
requests that change only files no compiler job reads: the documentation, Markdown files, GitHub metadata, other
workflows, and the license files. Pushes to `develop`, `master`, and `release/*` always run them. The documentation,
license, and path-filter checks run in [`repository_checks.yml`](../../.github/workflows/repository_checks.yml), which
has no path filter.
The filter starts with `'**'` and excludes these files explicitly, so a new file runs the compiler workflows unless it
is excluded on purpose. [`check_path_filters.py`](check_path_filters.py) holds the filter (`PATHS`) and checks that
- each of the three workflows has exactly this filter, with its own workflow file included,
- the CI targets that read excluded files (`UNFILTERED_TARGETS`) do not run in these workflows, and
- no file the filter keeps (sources, tests, build files, tools, workflows) mentions an excluded file or directory
outside a comment line, unless the mention is listed in `KNOWN_REFERENCES` with a reason.
When the check reports a mention, decide whether a compiler job reads that file:
- If it does, include the file in `PATHS` after the exclusions (as `package_managers.md` and the Swift example are) and
copy the new filter into the three workflows.
- If it does not, add the mention to `KNOWN_REFERENCES` with the reason.
Run the check with
```shell
python3 tools/check_path_filters/check_path_filters.py
```
To see which workflows a pull request would run, pass the changed files:
```shell
python3 tools/check_path_filters/check_path_filters.py --changed $(git diff --name-only origin/develop)
```
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
"""Check the path filters of the compiler workflows.
ubuntu.yml, macos.yml and windows.yml skip pull requests that change only files
no compiler job reads (documentation, Markdown files, GitHub metadata, license
files). The filter starts with '**' and excludes these files explicitly, so a
new file runs the workflows unless it is excluded on purpose.
The script checks that
- each of the three workflows has exactly the filter defined in PATHS below,
with its own workflow file included, and
- no tracked file that the filter keeps (sources, tests, build files, tools,
workflows) mentions a file the filter excludes, unless the mention is listed
in KNOWN_REFERENCES with the reason why it does not matter.
A mention found this way must either be added to PATHS as an included path
(the file is read by a compiler job) or to KNOWN_REFERENCES (it is not).
With --changed FILE..., the script prints instead which workflows a pull
request changing these files would run.
"""
import argparse
import os
import re
import subprocess # nosec B404
import sys
REPO_ROOT = os.path.normpath(os.path.join(sys.path[0], '..', '..'))
SELF = 'tools/check_path_filters/check_path_filters.py'
WORKFLOWS = ['.github/workflows/ubuntu.yml', '.github/workflows/macos.yml', '.github/workflows/windows.yml']
# The pull_request paths filter of the workflows above; '{workflow}' is replaced
# with the workflow's own file. GitHub applies the patterns in order: a file
# matching a later pattern is included ('pattern') or excluded ('!pattern').
PATHS = [
'**',
'!docs/**',
'!**.md',
'!.github/*',
'!.github/ISSUE_TEMPLATE/**',
'!.github/external_ci/**',
'!.github/workflows/**',
'!.reuse/**',
'!LICENSES/**',
'!CITATION.cff',
'!REUSE.toml',
# read by tools/check_build_options in the ci_meson_install job (ubuntu.yml)
'docs/mkdocs/docs/integration/package_managers.md',
# built by the swiftpm job (macos.yml)
'docs/mkdocs/docs/integration/swift/**',
'{workflow}',
]
# Mentions of excluded files that no job of the three workflows depends on:
# (file that mentions it, the mention, reason)
KNOWN_REFERENCES = [
('cmake/ci.cmake', 'docs/', 'ci_test_examples; runs in repository_checks.yml, which has no path filter'),
('cmake/ci.cmake', 'docs/mkdocs/', 'ci_test_build_documentation and ci_test_documentation_mermaid; they run in '
'repository_checks.yml, which has no path filter'),
('cmake/ci.cmake', 'docs/mkdocs/docs/examples/', 'indentation check of ci_test_amalgamation; the same check runs in '
'check_amalgamation.yml, which has no path filter'),
('Makefile', 'docs/mkdocs/docs/examples/', 'make pretty; runs in check_amalgamation.yml, which has no path filter'),
('Makefile', 'ChangeLog.md', 'make ChangeLog.md writes the file; no CI job runs it'),
('Makefile', 'tests/fuzzing.md', 'help text'),
('Makefile', 'tools/generate_natvis/README.md', 'help text'),
('tests/src/make_test_data_available.hpp', 'README.md', 'a file of the external test data repository'),
('tools/check_build_options/check_build_options.py', 'docs/', 'builds the path of package_managers.md, which '
'PATHS includes'),
]
# CI targets that read excluded files; they must run in a workflow without a path filter
UNFILTERED_TARGETS = ['ci_test_examples', 'ci_test_build_documentation', 'ci_test_documentation_mermaid',
'ci_test_amalgamation', 'ci_reuse_compliance']
def git_files(root):
# fixed command, no user input; includes new files not yet added, so a local run
# sees what CI sees after the commit
out = subprocess.check_output(['git', 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], cwd=root) # nosec B603, B607
return [f for f in out.decode('utf-8').split('\0') if f]
def glob_to_regex(pattern):
"""Translate a GitHub Actions path pattern to a regular expression."""
regex = ''
i = 0
while i < len(pattern):
if pattern.startswith('**', i):
regex += '.*'
i += 2
elif pattern[i] == '*':
regex += '[^/]*'
i += 1
elif pattern[i] == '?':
regex += '[^/]'
i += 1
else:
regex += re.escape(pattern[i])
i += 1
return re.compile(regex + r'\Z')
def compile_filter(paths, workflow):
return [(p.startswith('!'), glob_to_regex(p.lstrip('!').replace('{workflow}', workflow))) for p in paths]
def included(compiled, path):
result = False
for negated, regex in compiled:
if regex.match(path):
result = not negated
return result
def read_filter(root, workflow):
"""Return the pull_request paths list of a workflow file."""
with open(os.path.join(root, workflow), encoding='utf-8') as f:
lines = f.read().splitlines()
paths = None
in_pull_request = False
for line in lines:
if re.match(r'^ pull_request:', line):
in_pull_request = True
continue
if in_pull_request:
if re.match(r'^ \S', line) or re.match(r'^\S', line):
break
if re.match(r'^ paths:\s*$', line):
paths = []
continue
m = re.match(r"^ - '(.*)'\s*(#.*)?$", line)
if paths is not None and m:
paths.append(m.group(1))
elif paths is not None and line.strip() and not line.strip().startswith('#'):
break
return paths
TOKEN = re.compile(r'(["\'/]?)([\w.-]+(?:/[\w.-]+)*)')
COMMENT_LINE = re.compile(r'^[ \t]*(//|#|/?\*|--).*$', re.MULTILINE)
class Mentions:
"""Finds the words and paths in a text that name an excluded file or directory."""
def __init__(self, files, excluded, is_excluded):
self.files = set(files)
self.excluded = excluded
self.dirs = {'/'.join(f.split('/')[:n]) for f in files for n in range(1, f.count('/') + 1)}
# an excluded directory: a new file in it would be excluded
self.excluded_dirs = {d for d in self.dirs if is_excluded(d + '/new_file')}
# base names that only excluded files have, e.g. 'ChangeLog.md' but not 'CMakeLists.txt'
self.names = {os.path.basename(f) for f in excluded} - {os.path.basename(f) for f in files if f not in excluded}
def find(self, text):
"""Yield (offset, mention) for every word or path in text that names an excluded file or directory."""
for m in TOKEN.finditer(text):
parts = m.group(2).rstrip('/').split('/')
# a single word names a directory only in a path context, e.g. 'docs' in os.path.join('docs', ...)
path_context = len(parts) > 1 or m.group(1)
for i in range(len(parts)):
path = '/'.join(parts[i:])
if path in self.files:
if path in self.excluded:
yield m.start(2), path
break
if path in self.dirs and path_context:
if path in self.excluded_dirs:
yield m.start(2), path + '/'
break
else:
if parts[-1] in self.names and '.' in parts[-1]:
yield m.start(2), parts[-1]
def known(path, mention):
return any(path == file and mention == known_mention for file, known_mention, _ in KNOWN_REFERENCES)
def check_workflows(root):
"""Return the errors in the filters of the workflows and in the targets they run."""
errors = []
for workflow in WORKFLOWS:
expected = [p.replace('{workflow}', workflow) for p in PATHS]
actual = read_filter(root, workflow)
if actual != expected:
errors.append('{}: the pull_request paths filter differs from tools/check_path_filters/check_path_filters.py\n'
' expected: {}\n actual: {}'.format(workflow, expected, actual))
with open(os.path.join(root, workflow), encoding='utf-8') as f:
text = f.read()
errors += ['{}: runs {}, which reads excluded files; run it in repository_checks.yml'.format(workflow, target)
for target in UNFILTERED_TARGETS if re.search(r'\b' + target + r'\b', text)]
return errors
def scanned_text(root, path):
"""Return the text of a kept file without comment lines and without the filter itself, or None to skip it."""
# third-party code and Markdown files read nothing; this script lists the excluded files
if path.startswith('tests/thirdparty/') or path.endswith('.md') or path == SELF:
return None
try:
with open(os.path.join(root, path), encoding='utf-8') as f:
# a mention in a comment line reads nothing; keep the line numbers
text = COMMENT_LINE.sub('', f.read())
except (UnicodeDecodeError, OSError):
return None
if path in WORKFLOWS:
# the filter itself lists the excluded files
for p in PATHS:
text = text.replace(" - '{}'".format(p.replace('{workflow}', path)), '')
return text
def check_mentions(root, files, excluded, finder):
"""Return an error for every mention of an excluded file in a kept file that KNOWN_REFERENCES does not list."""
errors = []
for path in sorted(set(files) - excluded):
text = scanned_text(root, path)
reported = set()
for offset, mention in finder.find(text or ''):
if mention not in reported and not known(path, mention):
reported.add(mention)
line = text.count('\n', 0, offset) + 1
errors.append('{}:{}: mentions excluded {}; include it in PATHS if a compiler job reads it, '
'or add it to KNOWN_REFERENCES'.format(path, line, mention))
return errors
def check(root):
files = git_files(root)
union = [compile_filter(PATHS, w) for w in WORKFLOWS]
def is_excluded(path):
return not any(included(c, path) for c in union)
excluded = {f for f in files if is_excluded(f)}
errors = check_workflows(root) + check_mentions(root, files, excluded, Mentions(files, excluded, is_excluded))
for e in errors:
print(e)
if not errors:
print('path filters OK: {} of {} tracked files skip the compiler workflows'.format(len(excluded), len(files)))
return 1 if errors else 0
def explain(changed):
for workflow in WORKFLOWS:
compiled = compile_filter(PATHS, workflow)
hits = [f for f in changed if included(compiled, f)]
print('{}: {}'.format(workflow, 'runs ({})'.format(', '.join(hits[:3]) + (' ...' if len(hits) > 3 else ''))
if hits else 'skipped'))
return 0
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument('--root', default=REPO_ROOT, help='repository root')
parser.add_argument('--changed', nargs='+', metavar='FILE', help='show which workflows these changed files run')
args = parser.parse_args()
return explain(args.changed) if args.changed else check(args.root)
if __name__ == '__main__':
sys.exit(main())