From 069205946ac6153ebf3da8b5288b0c56444e42e3 Mon Sep 17 00:00:00 2001 From: Fam Zheng Date: Sat, 27 Dec 2025 17:26:44 +0000 Subject: [PATCH] Reduce parallelism to avoid overwhelming server with 404s --- emblem5/ai/fetch-scans.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emblem5/ai/fetch-scans.py b/emblem5/ai/fetch-scans.py index 8e1d800..9bed66f 100755 --- a/emblem5/ai/fetch-scans.py +++ b/emblem5/ai/fetch-scans.py @@ -65,7 +65,7 @@ class ScanDataFetcher(object): if sample_rate: fetch_backlog = random.sample(fetch_backlog, int(len(fetch_backlog) * sample_rate)) logger.info(f'fetch_backlog: {len(fetch_backlog)}') - pool = mp.Pool(mp.cpu_count() * 4) + pool = mp.Pool(min(8, mp.cpu_count())) counts = defaultdict(int) for r in tqdm(pool.imap_unordered(self.fetch_one_scan, fetch_backlog), total=len(fetch_backlog)): counts[r] += 1