diff --git a/export_swe_chat.py b/export_swe_chat.py index ce8a6fc..6580ea1 100644 --- a/export_swe_chat.py +++ b/export_swe_chat.py @@ -166,9 +166,12 @@ def select_candidates( ): funnel["rejectedMalformedOrEmpty"] += 1 continue - by_session[session_id].append( - Turn(session_id, turn_id, conversation_turn_number, turn_number, prompt) - ) + # Retain only the first three checked ordinals while streaming. This bounds + # memory by sessions × 3, not by every eligible prompt in the large config. + turns = by_session[session_id] + turns.append(Turn(session_id, turn_id, conversation_turn_number, turn_number, prompt)) + turns.sort(key=lambda turn: (turn.conversation_turn_number, turn.turn_number, turn.turn_id)) + del turns[3:] funnel["eligibleRows"] += 1 preliminary: list[Candidate] = []