diff --git a/emblem5/ai/fetch-scans.py b/emblem5/ai/fetch-scans.py index 8e1d800..9bed66f 100755 --- a/emblem5/ai/fetch-scans.py +++ b/emblem5/ai/fetch-scans.py @@ -65,7 +65,7 @@ class ScanDataFetcher(object): if sample_rate: fetch_backlog = random.sample(fetch_backlog, int(len(fetch_backlog) * sample_rate)) logger.info(f'fetch_backlog: {len(fetch_backlog)}') - pool = mp.Pool(mp.cpu_count() * 4) + pool = mp.Pool(min(8, mp.cpu_count())) counts = defaultdict(int) for r in tqdm(pool.imap_unordered(self.fetch_one_scan, fetch_backlog), total=len(fetch_backlog)): counts[r] += 1