Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,6 @@ Tools/check-c-api-docs/ @ZeroIntensity
# Fuzzing
Modules/_xxtestfuzz/ @python/fuzzers
Lib/test/test_xxtestfuzz.py @python/fuzzers
.github/workflows/reusable-cifuzz.yml @python/fuzzers

# Limited C API & Stable ABI
Doc/c-api/stable.rst @encukou
Expand Down
52 changes: 0 additions & 52 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -651,50 +651,6 @@ jobs:
run: |
"$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed

cifuzz:
# ${{ '' } is a hack to nest jobs under the same sidebar category.
name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation]
needs: build-context
if: >-
needs.build-context.outputs.run-ci-fuzz == 'true'
|| needs.build-context.outputs.run-ci-fuzz-stdlib == 'true'
permissions:
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
sanitizer:
- address
oss-fuzz-project-name:
- cpython3
- python3-libraries
include:
- sanitizer: undefined
oss-fuzz-project-name: cpython3
- sanitizer: memory
oss-fuzz-project-name: cpython3
exclude:
# Note that the 'no-exclude' sentinel below is to prevent
# an empty string value from excluding all jobs and causing
# GHA to create a 'default' matrix entry with all empty values.
- oss-fuzz-project-name: >-
${{
needs.build-context.outputs.run-ci-fuzz == 'true'
&& 'no-exclude'
|| 'cpython3'
}}
- oss-fuzz-project-name: >-
${{
needs.build-context.outputs.run-ci-fuzz-stdlib == 'true'
&& 'no-exclude'
|| 'python3-libraries'
}}
uses: ./.github/workflows/reusable-cifuzz.yml
with:
oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }}
sanitizer: ${{ matrix.sanitizer }}

all-required-green: # This job does nothing and is only used for the branch protection
name: All required checks pass
runs-on: ubuntu-latest
Expand All @@ -718,7 +674,6 @@ jobs:
- build-asan
- build-san
- cross-build-linux
- cifuzz
if: always()

steps:
Expand All @@ -731,7 +686,6 @@ jobs:
build-windows-msi,
build-ubuntu-ssltests,
test-hypothesis,
cifuzz,
allowed-skips: >-
${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }}
${{
Expand All @@ -749,12 +703,6 @@ jobs:
|| ''
}}
${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }}
${{
!fromJSON(needs.build-context.outputs.run-ci-fuzz)
&& !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib)
&& 'cifuzz,' ||
''
}}
${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }}
${{
!fromJSON(needs.build-context.outputs.run-ubuntu)
Expand Down
49 changes: 0 additions & 49 deletions .github/workflows/reusable-cifuzz.yml

This file was deleted.

8 changes: 0 additions & 8 deletions .github/workflows/reusable-context.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,6 @@ on: # yamllint disable-line rule:truthy
run-android:
description: Whether to run the Android tests
value: ${{ jobs.compute-changes.outputs.run-android }} # bool
run-ci-fuzz:
description: Whether to run the CIFuzz job for 'cpython' fuzzer
value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool
run-ci-fuzz-stdlib:
description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer
value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool
run-docs:
description: Whether to build the docs
value: ${{ jobs.compute-changes.outputs.run-docs }} # bool
Expand Down Expand Up @@ -64,8 +58,6 @@ jobs:
timeout-minutes: 10
outputs:
run-android: ${{ steps.changes.outputs.run-android }}
run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }}
run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }}
run-docs: ${{ steps.changes.outputs.run-docs }}
run-ios: ${{ steps.changes.outputs.run-ios }}
run-macos: ${{ steps.changes.outputs.run-macos }}
Expand Down
19 changes: 1 addition & 18 deletions Lib/test/test_tools/test_compute_changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@
from pathlib import Path
from unittest.mock import patch

from test.support import os_helper
from test.test_tools import basepath, skip_if_missing, imports_under_tool
from test.test_tools import skip_if_missing, imports_under_tool

skip_if_missing("build")

Expand All @@ -16,15 +15,13 @@
compute_changes = importlib.import_module("compute-changes")

process_changed_files = compute_changes.process_changed_files
is_fuzzable_library_file = compute_changes.is_fuzzable_library_file
Outputs = compute_changes.Outputs
ANDROID_DIRS = compute_changes.ANDROID_DIRS
IOS_DIRS = compute_changes.IOS_DIRS
MACOS_DIRS = compute_changes.MACOS_DIRS
WASI_DIRS = compute_changes.WASI_DIRS
RUN_TESTS_IGNORE = compute_changes.RUN_TESTS_IGNORE
UNIX_BUILD_SYSTEM_FILE_NAMES = compute_changes.UNIX_BUILD_SYSTEM_FILE_NAMES
LIBRARY_FUZZER_PATHS = compute_changes.LIBRARY_FUZZER_PATHS


class TestProcessChangedFiles(unittest.TestCase):
Expand All @@ -46,20 +43,6 @@ def test_docs(self):
self.assertTrue(result.run_docs)
self.assertFalse(result.run_tests)

def test_ci_fuzz_stdlib(self):
with os_helper.change_cwd(basepath):
for p in LIBRARY_FUZZER_PATHS:
with self.subTest(p=p):
if p.is_dir():
f = p / "file"
elif p.is_file():
f = p
else:
continue
result = process_changed_files({f})
self.assertTrue(result.run_ci_fuzz_stdlib)
self.assertTrue(is_fuzzable_library_file(f))

def test_android(self):
for d in ANDROID_DIRS:
with self.subTest(d=d):
Expand Down
97 changes: 2 additions & 95 deletions Tools/build/compute-changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
Path("Tools/build/regen-configure.sh"),
})

SUFFIXES_C_OR_CPP = frozenset({".c", ".h", ".cpp"})
SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"})

ANDROID_DIRS = frozenset({"Android"})
Expand All @@ -53,63 +52,9 @@
MACOS_DIRS = frozenset({"Mac"})
WASI_DIRS = frozenset({Path("Platforms", "WASI")})

LIBRARY_FUZZER_PATHS = frozenset({
# All C/CPP fuzzers.
Path("configure"),
Path(".github/workflows/reusable-cifuzz.yml"),
# ast
Path("Lib/ast.py"),
Path("Python/ast.c"),
# configparser
Path("Lib/configparser.py"),
# csv
Path("Lib/csv.py"),
Path("Modules/_csv.c"),
# decode
Path("Lib/encodings/"),
Path("Modules/_codecsmodule.c"),
Path("Modules/cjkcodecs/"),
Path("Modules/unicodedata.c"),
Path("Modules/unicodedata_db.h"),
# difflib
Path("Lib/difflib.py"),
# email
Path("Lib/email/"),
# html
Path("Lib/html/"),
Path("Lib/_markupbase.py"),
# http.client
Path("Lib/http/client.py"),
# json
Path("Lib/json/"),
Path("Modules/_json.c"),
# plist
Path("Lib/plistlib.py"),
# re
Path("Lib/re/"),
Path("Modules/_sre/"),
# tarfile
Path("Lib/tarfile.py"),
# tomllib
Path("Lib/tomllib/"),
# xml
Path("Lib/xml/"),
Path("Lib/_markupbase.py"),
Path("Modules/expat/"),
Path("Modules/pyexpat.c"),
# zipfile
Path("Lib/zipfile/"),
# zoneinfo
Path("Lib/zoneinfo/"),
Path("Modules/_zoneinfo.c"),
})


@dataclass(kw_only=True, slots=True)
class Outputs:
run_android: bool = False
run_ci_fuzz: bool = False
run_ci_fuzz_stdlib: bool = False
run_docs: bool = False
run_emscripten: bool = False
run_ios: bool = False
Expand Down Expand Up @@ -147,16 +92,6 @@ def compute_changes() -> None:
if outputs.run_windows_tests:
print("Run Windows tests")

if outputs.run_ci_fuzz:
print("Run CIFuzz tests")
else:
print("Branch too old for CIFuzz tests; or no C files were changed")

if outputs.run_ci_fuzz_stdlib:
print("Run CIFuzz tests for stdlib")
else:
print("Branch too old for CIFuzz tests; or no stdlib files were changed")

if outputs.run_docs:
print("Build documentation")

Expand Down Expand Up @@ -209,18 +144,8 @@ def get_file_platform(file: Path) -> str | None:
return None


def is_fuzzable_library_file(file: Path) -> bool:
return any(
(file.is_relative_to(needs_fuzz) and needs_fuzz.is_dir())
or (file == needs_fuzz and file.is_file())
for needs_fuzz in LIBRARY_FUZZER_PATHS
)


def process_changed_files(changed_files: Set[Path]) -> Outputs:
run_tests = False
run_ci_fuzz = False
run_ci_fuzz_stdlib = False
run_docs = False
run_windows_tests = False
run_windows_msi = False
Expand All @@ -234,8 +159,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc

if file.parent == GITHUB_WORKFLOWS_PATH:
if file.name in ("build.yml", "reusable-cifuzz.yml"):
run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = run_windows_tests = True
if file.name == "build.yml":
run_tests = run_windows_tests = True
has_platform_specific_change = False
continue
if file.name in ("reusable-docs.yml", "reusable-check-html-ids.yml"):
Expand Down Expand Up @@ -272,18 +197,6 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
if file not in UNIX_BUILD_SYSTEM_FILE_NAMES:
run_windows_tests = True

# The fuzz tests are pretty slow so they are executed only for PRs
# changing relevant files.
if file.suffix in SUFFIXES_C_OR_CPP:
run_ci_fuzz = True
if file.parts[:2] in {
("configure",),
("Modules", "_xxtestfuzz"),
}:
run_ci_fuzz = True
if not run_ci_fuzz_stdlib and is_fuzzable_library_file(file):
run_ci_fuzz_stdlib = True

# Check for changed documentation-related files
if doc_file:
run_docs = True
Expand Down Expand Up @@ -318,8 +231,6 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:

return Outputs(
run_android=run_android,
run_ci_fuzz=run_ci_fuzz,
run_ci_fuzz_stdlib=run_ci_fuzz_stdlib,
run_docs=run_docs,
run_emscripten=run_emscripten,
run_ios=run_ios,
Expand All @@ -336,10 +247,6 @@ def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs:
if not git_branch:
outputs.run_tests = True

# CIFuzz / OSS-Fuzz compatibility with older branches may be broken.
if git_branch != GITHUB_DEFAULT_BRANCH:
outputs.run_ci_fuzz = False

if os.environ.get("GITHUB_EVENT_NAME", "").lower() == "workflow_dispatch":
outputs.run_docs = True
outputs.run_windows_msi = True
Expand Down
Loading