Skip to content

add export_parquet command and pyarrow #1110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
DrillableBit wants to merge 5 commits into
base: main
Choose a base branch
from
Open

add export_parquet command and pyarrow #1110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e723150
add export_parquet command and pyarrow
DrillableBit May 11, 2026
c3d7225
remove delete cleanup, add description
DrillableBit May 14, 2026
50e09b5
remove logs
DrillableBit May 14, 2026
70a116f
Revert "remove logs"
DrillableBit May 14, 2026
bb7a361
Reapply "remove logs"
DrillableBit May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 3 additions & 47 deletions aiarena/api/arenaclient/common/ac_coordinator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
from aiarena.core.models import ArenaClient

import logging
import time

from django.db import connection, transaction
from django.db.models.signals import pre_save
Expand All @@ -33,40 +32,25 @@


logger = logging.getLogger(__name__)
loggerECS = logging.getLogger("aiarena")

threshold_logger = 3


class ACCoordinator:
"""Coordinates all the Arena Clients and which matches they play."""

@staticmethod
def next_requested_match(arenaclient: ArenaClient):
t = time.monotonic()
# REQUESTED MATCHES
with transaction.atomic():
match = matches.attempt_to_start_a_requested_match(arenaclient)

if match is not None:
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - ACCoordinator next_requested_match took %.3fs | match=%s",
time.monotonic() - t,
getattr(match, "id", None),
)
return match # a match was found - we're done
return None

@staticmethod
def next_competition_match(arenaclient: ArenaClient):
t = time.monotonic()
competition_ids = ACCoordinator._get_competition_priority_order()
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - ACCoordinator _get_competition_priority_order took %.3fs",
time.monotonic() - t,
)

for id in competition_ids:
competition = Competition.objects.get(id=id)
# This excludes non-trusted clients from competitions requiring trusted infrastructure
Expand All @@ -75,22 +59,9 @@ def next_competition_match(arenaclient: ArenaClient):
with transaction.atomic():
# this call will apply a select for update, so we do it inside an atomic block
has_matches = competitions.check_has_matches_to_play_and_apply_locks(competition)
if time.monotonic() - t > 3:
loggerECS.warning(
"Slow request - ACCoordinator check_has_matches_to_play_and_apply_locks took %.3fs | competition=%s",
time.monotonic() - t,
competition.id,
)
if has_matches:
try:
match = matches.start_next_match_for_competition(arenaclient, competition)
if time.monotonic() - t > 3:
loggerECS.warning(
"Slow request - ACCoordinator start_next_match_for_competition took %.3fs | competition=%s | match=%s",
time.monotonic() - t,
competition.id,
getattr(match, "id", None),
)
return match
except (
NoMaps,
Expand All @@ -106,20 +77,13 @@ def next_competition_match(arenaclient: ArenaClient):

@staticmethod
def next_new_match(arenaclient: ArenaClient):
t = time.monotonic()
requested_match = ACCoordinator.next_requested_match(arenaclient)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - ACCoordinator next_new_match next_requested_match took %.3fs",
time.monotonic() - t,
)
if requested_match is not None:
return requested_match
return ACCoordinator.next_competition_match(arenaclient)

@staticmethod
def next_match(arenaclient: ArenaClient, only_unfinished_matches: bool) -> Match | None:
t = time.monotonic()
if not config.LADDER_ENABLED:
raise LadderDisabled()

Expand All @@ -134,20 +98,12 @@ def next_match(arenaclient: ArenaClient, only_unfinished_matches: bool) -> Match
)
.order_by("round_id")
)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - ACCoordinator next_match Match.objects.only took %.3fs",
time.monotonic() - t,
)

if len(unfinished_matches) > 0:
return unfinished_matches[0] # todo: re-set started time?
if only_unfinished_matches:
return None # Return None so we don't try to start a new match
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - ACCoordinator next_match total took %.3fs",
time.monotonic() - t,
)

# Trying a new match
return ACCoordinator.next_new_match(arenaclient)

Expand Down
23 changes: 0 additions & 23 deletions aiarena/api/arenaclient/common/result_submission_handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import logging
import time
from dataclasses import dataclass
from typing import Any

Expand Down Expand Up @@ -30,23 +29,13 @@


logger = logging.getLogger(__name__)
loggerECS = logging.getLogger("aiarena")

threshold_logger = 3


def handle_result_submission(match_id, result_data):
t = time.monotonic()
with transaction.atomic():
match = Match.objects.prefetch_related(
Prefetch("matchparticipation_set", MatchParticipation.objects.all().select_related("bot"))
).get(id=match_id)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - handle_result_submission get Match took %.3fs | match_id=%s",
time.monotonic() - t,
match_id=match_id,
)

# validate result
result = SubmitResultResultSerializer(
Expand All @@ -60,12 +49,6 @@ def handle_result_submission(match_id, result_data):
}
)
result.is_valid(raise_exception=True)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - handle_result_submission validate result took %.3fs | match_id=%s",
time.monotonic() - t,
match_id=match_id,
)

# validate participants
p1_instance = match.matchparticipation_set.get(participant_number=1)
Expand Down Expand Up @@ -93,12 +76,6 @@ def handle_result_submission(match_id, result_data):
partial=True,
)
participant2.is_valid(raise_exception=True)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - handle_result_submission validate participants took %.3fs | match_id=%s",
time.monotonic() - t,
match_id=match_id,
)

result_submission = ResultSubmission(
match=match,
Expand Down
107 changes: 15 additions & 92 deletions aiarena/api/arenaclient/common/views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import logging
import time
from wsgiref.util import FileWrapper

from django.core.cache import cache
Expand Down Expand Up @@ -28,9 +27,6 @@


logger = logging.getLogger(__name__)
loggerECS = logging.getLogger("aiarena")

threshold_logger = 3


class MatchViewSet(viewsets.GenericViewSet):
Expand Down Expand Up @@ -59,83 +55,23 @@ def load_participants(self, match: Match):
)

def create(self, request, *args, **kwargs):
total_start = time.monotonic()
user_id = getattr(request.user, "id", "AnonymousUser")

try:
t = time.monotonic()
no_game_available = cache.get("NoGameAvailable", False)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - next-match cache.get took %.3fs | user=%s | no_game_available=%s",
time.monotonic() - t,
user_id,
no_game_available,
)

if request.user.is_arenaclient:
t = time.monotonic()
match = ACCoordinator.next_match(request.user.arenaclient, no_game_available)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - next-match ACCoordinator.next_match took %.3fs | user=%s | match=%s",
time.monotonic() - t,
user_id,
getattr(match, "id", None),
)

if match:
t = time.monotonic()
self.load_participants(match)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - next-match load_participants took %.3fs | user=%s | match=%s",
time.monotonic() - t,
user_id,
match.id,
)

t = time.monotonic()
serializer = self.get_serializer(match)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - next-match get_serializer took %.3fs | user=%s | match=%s",
time.monotonic() - t,
user_id,
match.id,
)

t = time.monotonic()
data = serializer.data
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - next-match serializer.data took %.3fs | user=%s | match=%s",
time.monotonic() - t,
user_id,
match.id,
)

return Response(data, status=status.HTTP_201_CREATED)

else:
if not no_game_available:
t = time.monotonic()
cache.set("NoGameAvailable", True, config.GAME_AVAILABLE_CACHE_TIME)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - next-match cache.set took %.3fs | user=%s",
time.monotonic() - t,
user_id,
)
no_game_available = cache.get("NoGameAvailable", False)

raise NoGameForClient()
if request.user.is_arenaclient:
match = ACCoordinator.next_match(request.user.arenaclient, no_game_available)

finally:
loggerECS.warning(
"Slow request - next-match create total %.3fs | user=%s",
time.monotonic() - total_start,
user_id,
)
if match:
self.load_participants(match)

serializer = self.get_serializer(match)

return Response(data=serializer.data, status=status.HTTP_201_CREATED)

else:
if not no_game_available:
cache.set("NoGameAvailable", True, config.GAME_AVAILABLE_CACHE_TIME)

raise NoGameForClient()

# todo: check match is in progress/bot is in this match
@action(
Expand Down Expand Up @@ -183,7 +119,6 @@ class ResultViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet):
swagger_schema = None # exclude this from swagger generation

def create(self, request, *args, **kwargs):
t = time.monotonic()
if not config.LADDER_ENABLED:
raise LadderDisabled()

Expand All @@ -210,19 +145,7 @@ def create(self, request, *args, **kwargs):
f"bot2_data: {serializer.validated_data.get('bot2_data')} "
f"bot2_tags: {serializer.validated_data.get('bot2_tags')} "
)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - ResultViewSet get_serializer took %.3fs",
time.monotonic() - t,
)
t = time.monotonic()
result = handle_result_submission(match_id, serializer.validated_data)
if time.monotonic() - t > threshold_logger:
loggerECS.warning(
"Slow request - ResultViewSet handle_result_submission took %.3fs",
time.monotonic() - t,
)
t = time.monotonic()
headers = self.get_success_headers(serializer.data)
return Response({"result_id": result.id}, status=status.HTTP_201_CREATED, headers=headers)

Expand Down
Loading
Loading