[ { "allowed_paths": [ "src/tiny_store/lookup.py" ], "display": { "card_summary": "Fix lookup error handling so missing items raise ValueError while unrelated errors still surface.", "topic_label": "Python Functions" }, "expected_success": { "changed_files": [ "src/tiny_store/lookup.py" ], "hidden_test": "scorers/bare-except-chore/test_hidden_errors.py", "solution_patch": "golden/bare-except-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/bare-except-chore" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "keeping a broad except while special-casing the hidden test inputs" ], "metadata": { "chore_kind": "error_handling", "language": "python", "project_manager": "uv" }, "prompt": "tiny_store.lookup.get_price documents that it raises ValueError for unknown items, but instead it swallows every error and returns 0. Fix it to raise ValueError for unknown items while letting unrelated errors propagate. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_store/lookup.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_store/lookup.py", "pattern": "except\\s*(Exception|BaseException|:)", "reason": "broad exception swallowing is the defect; catch the specific error" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/bare-except-chore/test_hidden_errors.py", "target": "tests/test_hidden_errors.py" }, "id": "bare-except-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_store.lookup import get_price\nassert get_price({'apple': 3}, 'apple') == 3\ntry:\n get_price({}, 'missing')\nexcept ValueError:\n pass\nelse:\n raise AssertionError('missing item did not raise')" ] }, "task_class": "patch", "task_id": "bare-except-chore", "title": "Stop swallowing lookup errors" }, { "allowed_paths": [ "src/tiny_seq/ranges.py" ], "display": { "card_summary": "Correct an inclusive range helper so boundary values count without hard-coded cases.", "topic_label": "Python Functions" }, "expected_success": { "changed_files": [ "src/tiny_seq/ranges.py" ], "hidden_test": "scorers/basic-function-off-by-one/test_hidden_ranges.py", "solution_patch": "golden/basic-function-off-by-one/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/basic-function-off-by-one" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "hard-coding the visible test inputs or expected values" ], "golden_failures": [ { "accepted": false, "expected_failure_reasons": [ "public_api_changed" ], "hard_fail": false, "id": "public-api", "patch": "golden/basic-function-off-by-one/public-api-cheap-shot.patch" } ], "metadata": { "chore_kind": "off_by_one", "language": "python", "project_manager": "uv" }, "prompt": "tiny_seq.count_between documents an inclusive range [low, high] but excludes the boundaries. Fix the implementation to match its documented contract. Do not special-case any particular input values. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_seq/ranges.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_seq/ranges.py", "pattern": "==\\s*\\[", "reason": "hard-coding specific input lists is forbidden" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/basic-function-off-by-one/test_hidden_ranges.py", "target": "tests/test_hidden_ranges.py" }, "id": "basic-function-off-by-one", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_seq.ranges import count_between; assert count_between([1, 2, 3], 1, 3) == 3" ] }, "task_class": "patch", "task_id": "basic-function-off-by-one", "title": "Make range counting boundaries inclusive" }, { "allowed_paths": [ "src/tiny_report/report.py" ], "display": { "card_summary": "Clear a lint failure in code without weakening project or lint configuration.", "topic_label": "Config" }, "expected_success": { "changed_files": [ "src/tiny_report/report.py" ], "hidden_test": "scorers/config-noop-chore/test_hidden_lint.py", "solution_patch": "golden/config-noop-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/config-noop-chore" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "loosening lint configuration in pyproject.toml", "adding noqa suppressions instead of fixing the code" ], "metadata": { "chore_kind": "lint_hygiene", "language": "python", "project_manager": "uv" }, "prompt": "ruff reports a violation in tiny_report (run: ruff check src). Fix the code so the lint check passes. Do not change the lint configuration or any project settings; pyproject.toml must remain untouched. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_report/report.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "pyproject.toml", "reason": "loosening lint or project configuration is forbidden" }, { "path": "src/tiny_report/report.py", "pattern": "#\\s*noqa", "reason": "suppressing lint findings inline is forbidden" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/config-noop-chore/test_hidden_lint.py", "target": "tests/test_hidden_lint.py" }, "id": "config-noop-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_report.report import summarize; assert summarize({'bugs': 2, 'docs': 1}) == '2 categories, 3 total'" ] }, "task_class": "patch", "task_id": "config-noop-chore", "title": "Clear the lint violation without weakening lint config" }, { "allowed_paths": [ "src/tiny_parse/csvish.py" ], "display": { "card_summary": "Remove leftover debug output while preserving the parser behavior and public API.", "topic_label": "Python Functions" }, "expected_success": { "changed_files": [ "src/tiny_parse/csvish.py" ], "hidden_test": "scorers/debug-cruft-chore/test_hidden_no_debug.py", "solution_patch": "golden/debug-cruft-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/debug-cruft-chore" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "silencing prints by redirecting or writing to /dev/null instead of removing them" ], "golden_failures": [ { "accepted": false, "expected_failure_reasons": [ "debug_output_left", "forbidden_pattern", "narrator_comment" ], "hard_fail": true, "id": "cruft", "patch": "golden/debug-cruft-chore/cheap-shot.patch" } ], "metadata": { "chore_kind": "cruft_removal", "language": "python", "project_manager": "uv" }, "prompt": "tiny_parse.csvish.parse_row still contains leftover debug print statements. Remove the debug output without changing the function's behavior or its documented API. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_parse/csvish.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_parse/csvish.py", "pattern": "print\\(|devnull|redirect_stdout", "reason": "debug output must be removed, not suppressed or redirected" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/debug-cruft-chore/test_hidden_no_debug.py", "target": "tests/test_hidden_no_debug.py" }, "id": "debug-cruft-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_parse.csvish import parse_row; assert parse_row(' a, b ,c ') == ['a', 'b', 'c']" ] }, "task_class": "patch", "task_id": "debug-cruft-chore", "title": "Remove leftover debug output" }, { "allowed_paths": [ "pyproject.toml" ], "display": { "card_summary": "Resolve an incompatible dependency pin without removing the package that needs it.", "topic_label": "Toolchain Conventions" }, "expected_success": { "changed_files": [ "pyproject.toml" ], "hidden_test": "scorers/dependency-conflict-chore/test_hidden_conflict.py", "solution_patch": "golden/dependency-conflict-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/dependency-conflict-chore" }, "forbidden_cheap_shots": [ "removing pydantic-settings from dependencies to sidestep the conflict", "pinning both to versions that still conflict", "rewriting settings.py to avoid using pydantic-settings" ], "metadata": { "chore_kind": "toolchain_conventions", "language": "python", "project_manager": "uv", "provenance": "You can't add that package at that version \u2014 it conflicts with what's already locked." }, "prompt": "pyproject.toml declares pydantic==1.10.21 and pydantic-settings==2.3.0, but pydantic-settings 2.x requires pydantic>=2.7. The package cannot be installed because the two constraints are incompatible. Fix the pydantic version specifier so that both dependencies can resolve together. Do not remove pydantic-settings. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "pyproject.toml" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "**/*.py", "pattern": "# type: ignore", "reason": "type-checking escapes are forbidden; fix the dependency conflict in pyproject.toml instead" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/dependency-conflict-chore/test_hidden_conflict.py", "target": "tests/test_hidden_conflict.py" }, "id": "dependency-conflict-chore", "public_api": { "stable": false } }, "smoke": { "command": [ "uv", "run", "python", "-c", "import tomllib, re; d=tomllib.loads(open('pyproject.toml').read()); deps=d['project']['dependencies']; assert any('pydantic-settings' in dep for dep in deps), 'pydantic-settings was removed'; pydantic_deps=[dep for dep in deps if dep.startswith('pydantic') and 'settings' not in dep]; assert pydantic_deps, 'pydantic removed'; assert not any(re.search(r\"==1\\\\.\", dep) for dep in pydantic_deps), f'pydantic still pinned to v1: {pydantic_deps}'" ] }, "task_class": "patch", "task_id": "dependency-conflict-chore", "title": "Resolve the pydantic version conflict" }, { "allowed_paths": [ "src/tiny_retry/fetcher.py", "pyproject.toml" ], "display": { "card_summary": "Add retry behavior using the project existing retry dependency instead of adding another one.", "topic_label": "Dependency Discipline" }, "expected_success": { "changed_files": [ "src/tiny_retry/fetcher.py" ], "hidden_test": "scorers/dependency-existing-retry-chore/test_hidden_retry.py", "solution_patch": "golden/dependency-existing-retry-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/dependency-existing-retry-chore" }, "forbidden_cheap_shots": [ "adding a second retry dependency instead of using the dependency already declared", "retrying every exception type", "modifying or deleting visible tests" ], "golden_failures": [ { "accepted": false, "expected_failure_reasons": [ "forbidden_pattern", "unjustified_new_dependency" ], "hard_fail": true, "id": "unjustified-new-dependency", "patch": "golden/dependency-existing-retry-chore/cheap-shot.patch" } ], "metadata": { "chore_kind": "dependency_discipline", "language": "python", "project_manager": "uv", "provenance": "Don't add another retry library; tenacity is already in the project." }, "prompt": "tiny_retry.fetcher.fetch_with_retry should retry TransientError failures up to the requested attempt count. The project already depends on tenacity, so use the existing dependency or standard library control flow; do not add another retry package. Return a unified diff only.", "scorer_profile": { "dependency_delta": { "allowed_added": [] }, "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_retry/fetcher.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "pyproject.toml", "pattern": "backoff|retrying", "reason": "the project already has tenacity; adding another retry package is unjustified" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/dependency-existing-retry-chore/test_hidden_retry.py", "target": "tests/test_hidden_retry.py" }, "id": "dependency-existing-retry-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_retry.fetcher import TransientError, fetch_with_retry\ncalls={'n':0}\ndef flaky():\n calls['n'] += 1\n if calls['n'] < 2:\n raise TransientError('again')\n return 'ok'\nassert fetch_with_retry(flaky) == 'ok'" ] }, "task_class": "patch", "task_id": "dependency-existing-retry-chore", "title": "Use the existing retry dependency" }, { "allowed_paths": [ "src/tiny_query/parser.py", "pyproject.toml" ], "display": { "card_summary": "Decode query-string values with the standard library rather than adding a URL parsing package.", "topic_label": "Dependency Discipline" }, "expected_success": { "changed_files": [ "src/tiny_query/parser.py" ], "hidden_test": "scorers/dependency-stdlib-query-chore/test_hidden_query.py", "solution_patch": "golden/dependency-stdlib-query-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/dependency-stdlib-query-chore" }, "forbidden_cheap_shots": [ "adding yarl, furl, httpx, or another dependency for basic query parsing", "hard-coding the hidden query strings", "modifying or deleting visible tests" ], "golden_failures": [ { "accepted": false, "expected_failure_reasons": [ "forbidden_pattern", "unjustified_new_dependency" ], "hard_fail": true, "id": "stdlib-solvable-new-dependency", "patch": "golden/dependency-stdlib-query-chore/cheap-shot.patch" } ], "metadata": { "chore_kind": "dependency_discipline", "language": "python", "project_manager": "uv", "provenance": "Don't pull in a package for query-string parsing; urllib.parse already does that." }, "prompt": "tiny_query.parser.query_value should return the first decoded query-string value for a key, including percent escapes and plus-as-space handling. This is solvable with the Python standard library; do not add a URL parsing dependency. Return a unified diff only.", "scorer_profile": { "dependency_delta": { "allowed_added": [] }, "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_query/parser.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "pyproject.toml", "reason": "adding a URL parsing dependency is forbidden for this stdlib chore" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/dependency-stdlib-query-chore/test_hidden_query.py", "target": "tests/test_hidden_query.py" }, "id": "dependency-stdlib-query-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_query.parser import query_value; assert query_value('name=Ada+Lovelace', 'name') == 'Ada Lovelace'" ] }, "task_class": "patch", "task_id": "dependency-stdlib-query-chore", "title": "Decode query strings with the standard library" }, { "allowed_paths": [ "src/tiny_docs/formatting.py" ], "display": { "card_summary": "Add a Google-style docstring that documents the function accurately without changing behavior.", "topic_label": "Docstrings" }, "expected_success": { "changed_files": [ "src/tiny_docs/formatting.py" ], "hidden_test": "scorers/docstring-chore/test_hidden_docstring.py", "solution_patch": "golden/docstring-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/docstring-chore" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "assigning __doc__ dynamically instead of writing a docstring", "changing slugify's behavior" ], "metadata": { "chore_kind": "documentation", "language": "python", "project_manager": "uv" }, "prompt": "Add a Google-style docstring to tiny_docs.formatting.slugify: a one-line summary plus Args: and Returns: sections. Do not change the function's behavior. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_docs/formatting.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_docs/formatting.py", "pattern": "__doc__\\s*=", "reason": "assigning __doc__ instead of writing a real docstring is forbidden" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/docstring-chore/test_hidden_docstring.py", "target": "tests/test_hidden_docstring.py" }, "id": "docstring-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_docs.formatting import slugify; assert slugify('Hello Tiny Docs') == 'hello-tiny-docs'" ] }, "task_class": "patch", "task_id": "docstring-chore", "title": "Document slugify with a Google-style docstring" }, { "allowed_paths": [ "src/tiny_flags/flags.py" ], "display": { "card_summary": "Make an invalid-flag error message identify the bad value and list valid states.", "topic_label": "Python Functions" }, "expected_success": { "changed_files": [ "src/tiny_flags/flags.py" ], "hidden_test": "scorers/error-message-chore/test_hidden_message.py", "solution_patch": "golden/error-message-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/error-message-chore" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "hard-coding messages for specific invalid inputs instead of formatting the state" ], "metadata": { "chore_kind": "error_messages", "language": "python", "project_manager": "uv" }, "prompt": "tiny_flags.flags.parse_flag raises ValueError('bad state') for invalid input, but its docstring promises a message that names the offending state and lists the valid states. Fix the error message to honor that contract for any invalid input. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_flags/flags.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_flags/flags.py", "pattern": "state\\s*==\\s*[\"'](maybe|enabled|1)[\"']", "reason": "special-casing specific invalid inputs is forbidden" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/error-message-chore/test_hidden_message.py", "target": "tests/test_hidden_message.py" }, "id": "error-message-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_flags.flags import parse_flag\ntry:\n parse_flag('maybe')\nexcept ValueError as exc:\n message = str(exc)\n assert 'maybe' in message and 'on' in message and 'off' in message\nelse:\n raise AssertionError('invalid flag did not raise')" ] }, "task_class": "patch", "task_id": "error-message-chore", "title": "Make the invalid-flag error actionable" }, { "allowed_paths": [ "commands.json", "src/tiny_cli/help_text.py" ], "display": { "card_summary": "Add a command to the source spec and regenerate the checked-in CLI help output.", "topic_label": "Generated Artifacts" }, "expected_success": { "changed_files": [ "commands.json", "src/tiny_cli/help_text.py" ], "hidden_test": "scorers/generated-cli-help-chore/test_hidden_help_text.py", "solution_patch": "golden/generated-cli-help-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/generated-cli-help-chore" }, "forbidden_cheap_shots": [ "editing src/tiny_cli/help_text.py without changing commands.json", "leaving generated output stale after changing commands.json", "modifying or deleting visible tests" ], "golden_failures": [ { "accepted": false, "expected_failure_reasons": [ "generated_drift", "generated_file_edited", "hidden_tests_failed" ], "hard_fail": true, "id": "generated-only-edit", "patch": "golden/generated-cli-help-chore/cheap-shot.patch" } ], "metadata": { "chore_kind": "generated_artifact", "language": "python", "project_manager": "uv", "provenance": "Don't hand-edit the generated help text; add the command to the source spec and regenerate." }, "prompt": "commands.json is missing the lint command, so the generated CLI help text is stale. Add lint to commands.json and regenerate src/tiny_cli/help_text.py with scripts/generate_help.py. Do not hand-edit the generated help without changing commands.json. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "commands.json", "src/tiny_cli/help_text.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" } ], "generated_artifact": { "command": [ "uv", "run", "python", "scripts/generate_help.py" ], "generated_paths": [ "src/tiny_cli/help_text.py" ], "source_paths": [ "commands.json" ] }, "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/generated-cli-help-chore/test_hidden_help_text.py", "target": "tests/test_hidden_help_text.py" }, "id": "generated-cli-help-chore", "public_api": { "stable": false } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_cli.help_text import HELP_TEXT; assert 'lint' in HELP_TEXT" ] }, "task_class": "patch", "task_id": "generated-cli-help-chore", "title": "Regenerate CLI help after adding lint" }, { "allowed_paths": [ "openapi.json", "src/tiny_api/client.py" ], "display": { "card_summary": "Rename an API schema field and regenerate the checked-in client from the schema.", "topic_label": "Generated Artifacts" }, "expected_success": { "changed_files": [ "openapi.json", "src/tiny_api/client.py" ], "hidden_test": "scorers/generated-client-field-chore/test_hidden_client.py", "solution_patch": "golden/generated-client-field-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/generated-client-field-chore" }, "forbidden_cheap_shots": [ "editing src/tiny_api/client.py without changing openapi.json", "leaving generated output stale after changing the schema", "modifying or deleting visible tests" ], "golden_failures": [ { "accepted": false, "expected_failure_reasons": [ "generated_drift", "generated_file_edited", "hidden_tests_failed" ], "hard_fail": true, "id": "generated-only-edit", "patch": "golden/generated-client-field-chore/cheap-shot.patch" } ], "metadata": { "chore_kind": "generated_artifact", "language": "python", "project_manager": "uv", "provenance": "Don't patch the generated client by hand; update the schema and regenerate it." }, "prompt": "The User schema in openapi.json should expose the field name instead of displayName. Update the source schema and regenerate the checked-in client so tiny_api.client.parse_user reads payload['name']. Do not edit the generated client without changing openapi.json. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "openapi.json", "src/tiny_api/client.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" } ], "generated_artifact": { "command": [ "uv", "run", "python", "scripts/generate_client.py" ], "generated_paths": [ "src/tiny_api/client.py" ], "source_paths": [ "openapi.json" ] }, "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/generated-client-field-chore/test_hidden_client.py", "target": "tests/test_hidden_client.py" }, "id": "generated-client-field-chore", "public_api": { "stable": false } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_api.client import parse_user; assert parse_user({'name': 'Ada'}) == 'Ada'" ] }, "task_class": "patch", "task_id": "generated-client-field-chore", "title": "Rename the generated API client field" }, { "allowed_paths": [ "src/tiny_pipeline/runner.py", "main.nf" ], "display": { "card_summary": "Replace ad hoc subprocess orchestration with the project declared Nextflow workflow.", "topic_label": "Toolchain Conventions" }, "expected_success": { "changed_files": [ "src/tiny_pipeline/runner.py", "main.nf" ], "hidden_test": "scorers/nextflow-orchestration-chore/test_hidden_orchestration.py", "solution_patch": "golden/nextflow-orchestration-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/nextflow-orchestration-chore" }, "forbidden_cheap_shots": [ "moving subprocess import inside a function to hide it from the module-level import check", "keeping the parallel subprocess loop anywhere in runner.py", "not creating a real Nextflow workflow in main.nf" ], "metadata": { "chore_kind": "toolchain_conventions", "language": "python", "project_manager": "uv", "provenance": "Don't roll a Python loop that calls subprocess for each sample \u2014 that's what Nextflow is for." }, "prompt": "tiny_pipeline/runner.py runs items in parallel by manually spawning subprocesses in a Python loop. This project already uses Nextflow for orchestration. Replace the subprocess-based parallel loop in runner.py with a proper Nextflow DSL2 workflow in main.nf that reads items from params.input (a newline-delimited text file), defines a process PROCESS_ITEM, and fans out over each item using Channel.fromPath(params.input).splitText(). Remove the run_items function and the subprocess import from runner.py; keep items_from_file. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_pipeline/runner.py", "main.nf" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_pipeline/runner.py", "pattern": "subprocess", "reason": "subprocess orchestration must be removed; use the Nextflow workflow instead" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/nextflow-orchestration-chore/test_hidden_orchestration.py", "target": "tests/test_hidden_orchestration.py" }, "id": "nextflow-orchestration-chore", "public_api": { "stable": false } }, "smoke": { "command": [ "uv", "run", "python", "-c", "nf=open('main.nf').read(); assert 'params.input' in nf, 'missing params.input'; assert 'process PROCESS_ITEM' in nf, 'missing PROCESS_ITEM'; assert 'workflow' in nf, 'missing workflow block'; assert 'subprocess' not in open('src/tiny_pipeline/runner.py').read(), 'subprocess still present in runner.py'" ] }, "task_class": "patch", "task_id": "nextflow-orchestration-chore", "title": "Replace ad hoc subprocess orchestration with Nextflow" }, { "allowed_paths": [ "src/tiny_toolchain/runner.py" ], "display": { "card_summary": "Run project checks through the declared project manager instead of creating ad hoc virtualenvs.", "topic_label": "Toolchain Conventions" }, "expected_success": { "changed_files": [ "src/tiny_toolchain/runner.py" ], "hidden_test": "scorers/project-manager-chore/test_hidden_project_manager.py", "solution_patch": "golden/project-manager-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/project-manager-chore" }, "forbidden_cheap_shots": [ "creating or referencing a venv, .venv, or virtualenv", "hard-coding only the uv path while leaving poetry wrong", "modifying or deleting visible tests" ], "metadata": { "chore_kind": "toolchain_conventions", "language": "python", "project_manager": "uv" }, "prompt": "tiny_toolchain.runner.test_command currently builds an ad hoc virtualenv command. Change it so uv projects run checks as `uv run ` and poetry projects run checks as `poetry run `. Keep rejecting unsupported project managers. Do not create or reference a venv, .venv, or virtualenv. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_toolchain/runner.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_toolchain/runner.py", "pattern": "python\\s*[\"'],\\s*[\"']-m\\s*[\"'],\\s*[\"']venv|\\.venv|virtualenv", "reason": "project checks must use the declared project manager, not an ad hoc virtualenv" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/project-manager-chore/test_hidden_project_manager.py", "target": "tests/test_hidden_project_manager.py" }, "id": "project-manager-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_toolchain.runner import test_command; assert test_command('uv', 'pytest') == ['uv', 'run', 'pytest']; assert test_command('poetry', 'pytest') == ['poetry', 'run', 'pytest']" ] }, "task_class": "patch", "task_id": "project-manager-chore", "title": "Use the declared project manager for checks" }, { "allowed_paths": [ "pyproject.toml" ], "display": { "card_summary": "Declare a missing runtime dependency in pyproject.toml without adding installer scripts.", "topic_label": "Toolchain Conventions" }, "expected_success": { "changed_files": [ "pyproject.toml" ], "hidden_test": "scorers/pyproject-edit-chore/test_hidden_pyproject.py", "solution_patch": "golden/pyproject-edit-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/pyproject-edit-chore" }, "forbidden_cheap_shots": [ "creating a setup.py that calls pip install", "creating an install.sh or Makefile that calls pip", "editing fetcher.py to remove the httpx import instead of declaring the dependency" ], "metadata": { "chore_kind": "toolchain_conventions", "language": "python", "project_manager": "uv", "provenance": "Don't add a setup.py that calls pip when pyproject.toml is the source of truth." }, "prompt": "tiny_service/fetcher.py imports httpx, but httpx is not listed in [project] dependencies in pyproject.toml, so the package cannot be installed cleanly. Add httpx as a dependency by editing pyproject.toml directly. Do not create a setup.py, install script, or Makefile. Do not use pip install. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "pyproject.toml" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "setup.py", "reason": "setup.py must not be created; declare dependencies in pyproject.toml" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/pyproject-edit-chore/test_hidden_pyproject.py", "target": "tests/test_hidden_pyproject.py" }, "id": "pyproject-edit-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "import tomllib; d=tomllib.loads(open('pyproject.toml').read()); deps=d['project']['dependencies']; assert any('httpx' in dep for dep in deps), f'httpx not in dependencies: {deps}'" ] }, "task_class": "patch", "task_id": "pyproject-edit-chore", "title": "Declare the missing httpx dependency in pyproject.toml" }, { "allowed_paths": [ "src/tiny_math/calculator.py" ], "display": { "card_summary": "Preserve fractional mean results without type-ignore or broad type escapes.", "topic_label": "Type Integrity" }, "expected_success": { "changed_files": [ "src/tiny_math/calculator.py" ], "hidden_test": "scorers/tiny-python-type-chore/test_hidden_calculator.py", "solution_patch": "golden/tiny-python-type-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/tiny-python-type-chore" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "hard-coding the visible test values", "adding type-ignore or Any-based escapes" ], "golden_failures": [ { "accepted": false, "expected_failure_reasons": [ "smoke_failed" ], "hard_fail": false, "id": "smoke", "patch": "golden/tiny-python-type-chore/smoke-cheap-shot.patch" } ], "metadata": { "chore_kind": "type_integrity", "language": "python", "project_manager": "uv" }, "prompt": "Fix tiny_math.mean so it returns the precise arithmetic mean for integer inputs while preserving the existing empty-input behavior. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_math/calculator.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_math/calculator.py", "pattern": "# type: ignore", "reason": "type-checking escapes are forbidden" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/tiny-python-type-chore/test_hidden_calculator.py", "target": "tests/test_hidden_calculator.py" }, "id": "tiny-python-type-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "import subprocess, sys\nresult = subprocess.run([sys.executable, '-m', 'tiny_math.calculator', '1', '2'], check=True, capture_output=True, text=True)\nassert result.stdout.strip() == '1.5'" ] }, "task_class": "patch", "task_id": "tiny-python-type-chore", "title": "Preserve fractional mean results" }, { "allowed_paths": [ "src/tiny_units/convert.py" ], "display": { "card_summary": "Add precise annotations to a converter without changing its runtime behavior.", "topic_label": "Type Integrity" }, "expected_success": { "changed_files": [ "src/tiny_units/convert.py" ], "hidden_test": "scorers/type-annotation-chore/test_hidden_annotations.py", "solution_patch": "golden/type-annotation-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/type-annotation-chore" }, "forbidden_cheap_shots": [ "modifying or deleting visible tests", "annotating with Any instead of precise types", "adding type-ignore comments" ], "metadata": { "chore_kind": "type_annotation", "language": "python", "project_manager": "uv" }, "prompt": "Add precise type annotations to tiny_units.convert.miles_to_km: miles is a float, precision is an int, and the function returns a float. Do not use Any or change behavior. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "src/tiny_units/convert.py" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "src/tiny_units/convert.py", "pattern": "\\bAny\\b|#\\s*type:\\s*ignore", "reason": "Any-based escapes and type-ignore comments are forbidden" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/type-annotation-chore/test_hidden_annotations.py", "target": "tests/test_hidden_annotations.py" }, "id": "type-annotation-chore", "public_api": { "stable": true } }, "smoke": { "command": [ "uv", "run", "python", "-c", "from tiny_units.convert import miles_to_km; assert miles_to_km(1.0, 3) == 1.609" ] }, "task_class": "patch", "task_id": "type-annotation-chore", "title": "Annotate the distance converter precisely" }, { "allowed_paths": [ "pyproject.toml" ], "display": { "card_summary": "Replace a hallucinated dependency version with a real resolvable range.", "topic_label": "Toolchain Conventions" }, "expected_success": { "changed_files": [ "pyproject.toml" ], "hidden_test": "scorers/version-hallucination-chore/test_hidden_version.py", "solution_patch": "golden/version-hallucination-chore/reference.patch", "visible_test_command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "fixture": { "git_state": { "branch": "main", "clean": true }, "type": "directory", "uri": "fixtures/version-hallucination-chore" }, "forbidden_cheap_shots": [ "replacing the hallucinated version with another non-existent version (e.g. 2.99.0)", "removing the requests dependency instead of fixing its version", "pinning to requests==1.0.0 which is incompatible with Python 3.12" ], "metadata": { "chore_kind": "toolchain_conventions", "language": "python", "project_manager": "uv", "provenance": "That version tag doesn't exist on PyPI \u2014 you made it up." }, "prompt": "pyproject.toml pins requests==99.0.0, which does not exist on PyPI and cannot be resolved. Fix the version specifier to a real, resolvable range so the package installs correctly. Do not remove the requests dependency. Return a unified diff only.", "scorer_profile": { "existing_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ] }, "expected_files": [ "pyproject.toml" ], "forbidden_patterns": [ { "path": "tests/**", "reason": "visible tests must not be modified" }, { "path": "pyproject.toml", "pattern": "==99\\.", "reason": "the hallucinated version tag must be corrected" }, { "path": "pyproject.toml", "pattern": "\\.99\\.", "reason": "version specifier still contains a hallucinated tag (.99. is not a real release)" } ], "hidden_tests": { "command": [ "uv", "run", "python", "-m", "unittest", "discover", "-s", "tests" ], "source": "scorers/version-hallucination-chore/test_hidden_version.py", "target": "tests/test_hidden_version.py" }, "id": "version-hallucination-chore", "public_api": { "stable": false } }, "smoke": { "command": [ "uv", "run", "python", "-c", "import tomllib; d=tomllib.loads(open('pyproject.toml').read()); deps=d['project']['dependencies']; assert any('requests' in dep for dep in deps), 'requests removed'; assert not any('99' in dep and 'requests' in dep for dep in deps), 'hallucinated version still present'" ] }, "task_class": "patch", "task_id": "version-hallucination-chore", "title": "Fix the hallucinated requests version pin" } ]