From 7bc1c3308f0aa2caf59bc0e23d02ed553ddda5ec Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 May 2019 11:48:59 -0400 Subject: [PATCH 1/6] don't run more than 61 workers on Windows --- black.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/black.py b/black.py index e7dce5bf2ab..41c42f943d5 100644 --- a/black.py +++ b/black.py @@ -441,7 +441,11 @@ def main( ) else: loop = asyncio.get_event_loop() - executor = ProcessPoolExecutor(max_workers=os.cpu_count()) + worker_count = os.cpu_count() + if sys.platform == "win32": + # Work around https://bugs.python.org/issue26903 + worker_count = min(worker_count, 61) + executor = ProcessPoolExecutor(max_workers=worker_count) try: loop.run_until_complete( schedule_formatting( From 59457eed693ccda65424a173a323c1c76b26562f Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 May 2019 11:59:19 -0400 Subject: [PATCH 2/6] split off a function --- black.py | 54 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/black.py b/black.py index 41c42f943d5..0018bd22f4c 100644 --- a/black.py +++ b/black.py @@ -440,26 +440,10 @@ def main( report=report, ) else: - loop = asyncio.get_event_loop() - worker_count = os.cpu_count() - if sys.platform == "win32": - # Work around https://bugs.python.org/issue26903 - worker_count = min(worker_count, 61) - executor = ProcessPoolExecutor(max_workers=worker_count) - try: - loop.run_until_complete( - schedule_formatting( - sources=sources, - fast=fast, - write_back=write_back, - mode=mode, - report=report, - loop=loop, - executor=executor, - ) - ) - finally: - shutdown(loop) + reformat_many( + sources=sources, fast=fast, write_back=write_back, mode=mode, report=report + ) + if verbose or not quiet: bang = "💥 💔 💥" if report.return_code else "✨ 🍰 ✨" out(f"All done! {bang}") @@ -467,6 +451,36 @@ def main( ctx.exit(report.return_code) +def reformat_many( + sources: List[Path], + fast: bool, + write_back: WriteBack, + mode: FileMode, + report: "Report", +) -> None: + """Reformat multiple files using a ProcessPoolExecutor.""" + loop = asyncio.get_event_loop() + worker_count = os.cpu_count() + if sys.platform == "win32": + # Work around https://bugs.python.org/issue26903 + worker_count = min(worker_count, 61) + executor = ProcessPoolExecutor(max_workers=worker_count) + try: + loop.run_until_complete( + schedule_formatting( + sources=sources, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + loop=loop, + executor=executor, + ) + ) + finally: + shutdown(loop) + + def reformat_one( src: Path, fast: bool, write_back: WriteBack, mode: FileMode, report: "Report" ) -> None: From 624aa945542128967145eb6e2b5e55bcbca14da5 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 May 2019 12:01:57 -0400 Subject: [PATCH 3/6] move functions around --- black.py | 60 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/black.py b/black.py index 0018bd22f4c..016953621c8 100644 --- a/black.py +++ b/black.py @@ -451,36 +451,6 @@ def main( ctx.exit(report.return_code) -def reformat_many( - sources: List[Path], - fast: bool, - write_back: WriteBack, - mode: FileMode, - report: "Report", -) -> None: - """Reformat multiple files using a ProcessPoolExecutor.""" - loop = asyncio.get_event_loop() - worker_count = os.cpu_count() - if sys.platform == "win32": - # Work around https://bugs.python.org/issue26903 - worker_count = min(worker_count, 61) - executor = ProcessPoolExecutor(max_workers=worker_count) - try: - loop.run_until_complete( - schedule_formatting( - sources=sources, - fast=fast, - write_back=write_back, - mode=mode, - report=report, - loop=loop, - executor=executor, - ) - ) - finally: - shutdown(loop) - - def reformat_one( src: Path, fast: bool, write_back: WriteBack, mode: FileMode, report: "Report" ) -> None: @@ -515,6 +485,36 @@ def reformat_one( report.failed(src, str(exc)) +def reformat_many( + sources: List[Path], + fast: bool, + write_back: WriteBack, + mode: FileMode, + report: "Report", +) -> None: + """Reformat multiple files using a ProcessPoolExecutor.""" + loop = asyncio.get_event_loop() + worker_count = os.cpu_count() + if sys.platform == "win32": + # Work around https://bugs.python.org/issue26903 + worker_count = min(worker_count, 61) + executor = ProcessPoolExecutor(max_workers=worker_count) + try: + loop.run_until_complete( + schedule_formatting( + sources=sources, + fast=fast, + write_back=write_back, + mode=mode, + report=report, + loop=loop, + executor=executor, + ) + ) + finally: + shutdown(loop) + + async def schedule_formatting( sources: Set[Path], fast: bool, From 642df75a07e65ecb855ff9f55025007a4fa23d6e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 May 2019 12:02:49 -0400 Subject: [PATCH 4/6] add to CHANGELOG --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index afc9e8e8087..bb9f0cdffb7 100644 --- a/README.md +++ b/README.md @@ -986,6 +986,8 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md). ### 19.5b0 +* don't crash when run on a Windows machine with more than 62 cores (#838) + * remove unnecessary parentheses around `yield` expressions (#834) * add parentheses around long tuples in unpacking assignments (#832) From 361d7f47f8a80c38cbd4b5cdaa7094503ed73860 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 May 2019 12:06:15 -0400 Subject: [PATCH 5/6] it's a Set, not a List --- black.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/black.py b/black.py index 016953621c8..18d60c02d6b 100644 --- a/black.py +++ b/black.py @@ -486,7 +486,7 @@ def reformat_one( def reformat_many( - sources: List[Path], + sources: Set[Path], fast: bool, write_back: WriteBack, mode: FileMode, From efa9ea1cb1adab4ca82851d998d20acd3389e9f3 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 May 2019 12:19:36 -0400 Subject: [PATCH 6/6] apparently it also fails with 62 cores --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb9f0cdffb7..28c4fbe0d84 100644 --- a/README.md +++ b/README.md @@ -986,7 +986,7 @@ More details can be found in [CONTRIBUTING](CONTRIBUTING.md). ### 19.5b0 -* don't crash when run on a Windows machine with more than 62 cores (#838) +* don't crash when run on a Windows machine with more than 61 cores (#838) * remove unnecessary parentheses around `yield` expressions (#834)