Skip to content

Commit

Permalink
Added country rank
Browse files Browse the repository at this point in the history
Added codecracker.arhn.in resource
Skip minor updates when resource have ongoning contest
Added last_rating_activity
Parse codeforces handles for icpc standings
  • Loading branch information
aropan committed Apr 20, 2024
1 parent 4dee428 commit c3d2b9e
Show file tree
Hide file tree
Showing 73 changed files with 1,745 additions and 236 deletions.
1 change: 1 addition & 0 deletions cron
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ DJANGO_ENV_FILE=.env.prod
*/1 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_CHECKING_LOGS /usr/src/clist/run-manage.bash check_logs
# 32 1 * * wed env MONITOR_NAME=SENTRY_CRON_MONITOR_REINDEX /usr/src/clist/run-manage.bash reindex
*/15 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SET_ACCOUNT_RANK /usr/src/clist/run-manage.bash set_account_rank
*/20 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_SET_COUNTRY_RANK /usr/src/clist/run-manage.bash set_country_rank
15 * * * * env MONITOR_NAME=SENTRY_CRON_MONITOR_UPDATE_AUTO_RATING /usr/src/clist/run-manage.bash update_auto_rating

# # 58 3 14-20 * * [ "$(date '+\%u')" -eq 4 ] && cd $PROJECT_DIR && run-one ./manage.py runscript calculate_account_contests >logs/command/calculate_account_contests.log 2>&1
Expand Down
22 changes: 22 additions & 0 deletions legacy/module/codecracker.arhn.in/index.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?php
require_once dirname(__FILE__) . '/../../config.php';

$data = curlexec($URL, null, array("http_header" => array('content-type: application/json'), "json_output" => 1));

foreach ($data as $_ => $c) {
$contests[] = array(
'start_time' => $c['start_time'],
'end_time' => $c['end_time'],
'title' => $c['contest_name'],
'url' => url_merge($HOST_URL, '/' . $c['contest_code'] . '/'),
'host' => $HOST,
'rid' => $RID,
'timezone' => $TIMEZONE,
'key' => $c['contest_code'],
);
}

if ($RID === -1) {
print_r($contests);
}
?>
2 changes: 1 addition & 1 deletion legacy/module/icpc.baylor.edu/index.php
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
$diff = abs($duration_time - $duration_in_secs);
if ($diff < $opt) {
$opt = $diff;
$duration = $duration_time;
$duration = $duration_time / 60;
$opt_start_time = trim($start_time);
}
}
Expand Down
18 changes: 12 additions & 6 deletions legacy/module/toph.co/index.php
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
<?php
require_once dirname(__FILE__) . "/../../config.php";

$page = curlexec($URL);

preg_match_all('#<a[^>]*href="?(?P<url>[^">]*)"?[^>]*>\s*<h2[^>]*>(?P<title>[^<]*)</h2>\s*</a>#', $page, $matches, PREG_SET_ORDER);

$parse_full = isset($_GET['parse_full_list']);

$url = $parse_full? url_merge($URL, 'all') : $URL;
$page = curlexec($url);
preg_match_all('#<a[^>]*href="?(?P<url>/c/[^">/\#]*)/?"?>\s*(?:<h2[^>]*>)?(?P<title>[^<]*)</#', $page, $matches, PREG_SET_ORDER);
if ($parse_full) {
for ($n_page = 2;; $n_page += 1) {
preg_match_all('#<td[^>]*><a[^>]*href="?(?P<url>[^">]*)"?[^>]*>\s*(?P<title>[^<]*)</a>#', $page, $m, PREG_SET_ORDER);
preg_match_all('#<a[^>]*href="?(?P<url>/c/[^">/\#]*)/?"?>\s*(?:<h2[^>]*>)?(?P<title>[^<]*)</#', $page, $m, PREG_SET_ORDER);
$matches = array_merge($matches, $m);

if (!preg_match('#<a[^>]*href="?(?P<href>[^">]*)"?>' . $n_page . '</a>#', $page, $match)) {
Expand All @@ -18,12 +18,18 @@
}
}

$seen = array();
foreach ($matches as $match) {
$url = url_merge($URL, $match['url']);
if (isset($seen[$url])) {
continue;
}
$seen[$url] = true;

$title = $match['title'];

$page = curlexec($url);
if (!preg_match('#will start [^<]*<[^>]*(?:data-time|data-timestamp)=(?P<start_time>[0-9]+)[^>]*>(?:[^<]*<[^>]*>[^<]*</[^>]*>)*</[^>]*>[^<]*will run for <[^>]*>(?P<duration>[^<]*)</#', $page, $match)) {
if (!preg_match('#(will start|started on) [^<]*<[^>]*(?:data-time|data-timestamp)=(?P<start_time>[0-9]+)[^>]*>(?:[^<]*<[^>]*>[^<]*</[^>]*>)*</[^>]*>[^<]*(will run|ran) for <[^>]*>(?P<duration>[^<]*)</#', $page, $match)) {
continue;
}

Expand Down
13 changes: 7 additions & 6 deletions legacy/update.php
Original file line number Diff line number Diff line change
Expand Up @@ -343,14 +343,15 @@ function($m) {

$contest = $db->escapeArray($contest);

if (isset($contest['old_key']) && $contest['old_key']) {
if (isset($contest['old_key'])) {
$old_key = $contest['old_key'];
unset($contest['old_key']);
$key = $contest['key'];

$old_update = "$update and key = '${old_key}'";
if (!$db->query("UPDATE clist_contest SET key = '$key' WHERE $old_update", true)) {
$db->query("DELETE FROM clist_contest WHERE $old_update", true);
if ($old_key) {
$key = $contest['key'];
$old_update = "$update and key = '${old_key}'";
if (!$db->query("UPDATE clist_contest SET key = '$key' WHERE $old_update", true)) {
$db->query("DELETE FROM clist_contest WHERE $old_update", true);
}
}
}

Expand Down
1 change: 0 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ django-tastypie==0.14.5
django-user-agents==0.4.0
django-webpush==0.3.5
git+https://github.com/appfluence/django-tastypie-swagger@763f8f027ec5cc6e20c33812cd8dec5b32404583
git+https://github.com/orcasgit/django-tastypie-oauth@01c40de7b86ae3c6cc9e9463ec9bcecbb22b897e
git+https://github.com/shtalinberg/django-el-pagination@c0885a0625c1698efba1076ae8cc9764763bcb10
pytest-django==4.5.2
phonenumbers==8.13.17
Expand Down
16 changes: 11 additions & 5 deletions src/clist/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from clist.models import Banner, Contest, ContestSeries, Problem, ProblemTag, Resource
from pyclist.admin import BaseModelAdmin, admin_register
from ranking.management.commands.parse_statistic import Command as parse_stat
from ranking.models import Rating, Module
from ranking.models import Module, Rating


@admin_register(Contest)
Expand Down Expand Up @@ -60,7 +60,8 @@ def parse_statistic(self, request, queryset):
['Secury information', {'fields': ['key']}],
['Addition information', {'fields': ['was_auto_added', 'auto_updated', 'n_statistics', 'has_hidden_results',
'calculate_time', 'info', 'invisible', 'is_rated', 'with_medals',
'related', 'series', 'allow_updating_statistics_for_participants']}],
'related', 'merging_contests', 'series',
'allow_updating_statistics_for_participants']}],
['Timing', {'fields': ['parsed_time', 'notification_timing', 'statistic_timing', 'rating_prediction_timing',
'created', 'modified', 'updated']}],
['Rating', {'fields': ['rating_prediction_hash', 'has_fixed_rating_prediction_field',
Expand Down Expand Up @@ -128,7 +129,10 @@ def queryset(self, request, queryset):
'n_accounts', 'n_contests']}],
['Parse information', {'fields': ['regexp', 'path', 'parse_url', 'timezone', 'auto_remove_started']}],
['Calendar information', {'fields': ['color', 'uid']}],
['Rating information', {'fields': ['has_rating_history', 'avg_rating', 'rating_update_time', 'rank_update_time',
['Rating information', {'fields': ['has_rating_history', 'has_country_rating',
'avg_rating', 'n_rating_accounts',
'rating_update_time', 'rank_update_time',
'contest_update_time', 'country_rank_update_time',
'ratings', 'rating_prediction']}],
['Account information', {'fields': ['has_accounts_infos_update', 'has_multi_account',
'has_account_verification', 'has_standings_renamed_account',
Expand All @@ -141,8 +145,9 @@ def queryset(self, request, queryset):
'_has_rating', '_has_profile_url', '_has_problem_rating', '_has_accounts_infos_update',
'_has_multi_account', '_has_standings_renamed_account', '_has_upsolving', '_has_verification']
search_fields = ['host', 'url']
list_filter = ['has_rating_history', HasProfileListFilter, 'enable', 'timezone', 'has_problem_rating',
'has_accounts_infos_update', 'has_multi_account', 'has_upsolving', 'has_account_verification']
list_filter = ['has_rating_history', 'has_country_rating', HasProfileListFilter, 'enable', 'timezone',
'has_problem_rating', 'has_accounts_infos_update', 'has_multi_account', 'has_upsolving',
'has_account_verification']

def _has_profile_url(self, obj):
return bool(obj.profile_url)
Expand Down Expand Up @@ -199,6 +204,7 @@ def get_readonly_fields(self, request, obj=None):

class ModuleInline(admin.StackedInline):
model = Module
extra = 0

inlines = (ModuleInline, )

Expand Down
172 changes: 172 additions & 0 deletions src/clist/api/authentication.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
import logging
import json
import six

from django.contrib.auth.models import AnonymousUser
from django.utils import timezone
from oauth2_provider.models import AccessToken
from tastypie.authentication import Authentication

"""
This is a simple OAuth 2.0 authentication model for tastypie
Dependencies (one of these):
- django-oauth-toolkit: https://github.com/evonove/django-oauth-toolkit
- django-oauth2-provider: https://github.com/caffeinehit/django-oauth2-provider
"""

log = logging.getLogger('tastypie_oauth')


class OAuthError(RuntimeError):
"""Generic exception class."""
def __init__(self, message='OAuth error occured.'):
self.message = message


class OAuth20Authentication(Authentication):
"""
OAuth authenticator.
This Authentication method checks for a provided HTTP_AUTHORIZATION
and looks up to see if this is a valid OAuth Access Token
"""
def __init__(self, realm='API'):
self.realm = realm

def is_authenticated(self, request, **kwargs):
"""
Verify 2-legged oauth request. Parameters accepted as
values in the "Authorization" header, as a GET request parameter,
or in a POST body.
"""
log.info("OAuth20Authentication")
try:
key = request.GET.get('oauth_consumer_key')
if not key:
for header in ['Authorization', 'HTTP_AUTHORIZATION']:
auth_header_value = request.META.get(header)
if auth_header_value and ' ' in auth_header_value:
key = auth_header_value.split(' ', 1)[1]
break
if not key and request.method == 'POST':
if request.META.get('CONTENT_TYPE') == 'application/json':
decoded_body = request.body.decode('utf8')
try:
key = json.loads(decoded_body)['oauth_consumer_key']
except (ValueError, KeyError):
pass
if not key:
log.info('OAuth20Authentication. No consumer_key found.')
return None
"""
If verify_access_token() does not pass, it will raise an error
"""
token = self.verify_access_token(key, request, **kwargs)

# If OAuth authentication is successful, set the request user to
# the token user for authorization
request.user = token.user or AnonymousUser()

# If OAuth authentication is successful, set oauth_consumer_key on
# request in case we need it later
request.META['oauth_consumer_key'] = key
return True
except KeyError:
log.exception("Error in OAuth20Authentication.")
request.user = AnonymousUser()
return False
except OAuthError:
return False
except Exception:
log.exception("Error in OAuth20Authentication.")
return False

def verify_access_token(self, key, request, **kwargs):
# Check if key is in AccessToken key
try:
token = AccessToken.objects.get(token=key)

# Check if token has expired
if token.expires < timezone.now():
raise OAuthError('AccessToken has expired.')
except AccessToken.DoesNotExist:
raise OAuthError("AccessToken not found at all.")

log.info('Valid access')
return token


class OAuth2ScopedAuthentication(OAuth20Authentication):
def __init__(self, realm="API", post=None, get=None, patch=None, put=None, delete=None, use_default=True, **kwargs):
"""
https://tools.ietf.org/html/rfc6749
get, post, patch and put is desired to be a scope or a list of scopes or None
if get is None, it will default to post
if delete is None, it will default to post
if both patch and put are None, they are all default to post
if one of patch or put is None, the two will default to the one that is not None
You can turn this overriding behavior off entirely by specifying use_default=False, but then remember
that None means no scope requirement is specified for that http method
the list of scopes should have a logic "or" between them
e.g. get=("a b","c") for oauth2-toolkit means "GET method requires scope 'a b'('a' and 'b') or scope 'c' "
get=(a|b,c) is the corresponding form for oauth2-provider, where a,b,c should be some constants you
defined in your settings
Note: for oauth2-toolkit, you have to provide a space seperated string of combination of scopes
you can also specify only one scope(instead of a list), and that scope will the only scope that has
permission to the according method
"""
super(OAuth2ScopedAuthentication, self).__init__(realm)
self.POST = post
if use_default:
self.GET = get or post
self.DELETE = delete or post
if not patch and not put:
self.PATCH = self.PUT = post
elif not patch or not put:
self.PATCH = self.PUT = (put or patch)
else:
self.PATCH = patch
self.PUT = put
else:
self.GET = get
self.PUT = put
self.PATCH = patch
self.DELETE = delete

def verify_access_token(self, key, request, **kwargs):
token = super(OAuth2ScopedAuthentication, self).verify_access_token(key, request, **kwargs)
if not self.check_scope(token, request):
raise OAuthError("AccessToken does not meet scope requirement")
# TODO: Return the actual scope granted if it is different
return token

def check_scope(self, token, request):
http_method = request.method
if not hasattr(self, http_method):
raise OAuthError("HTTP method is not recognized")
required_scopes = getattr(self, http_method)
# a None scope means always allowed
if required_scopes is None:
return True
"""
The required scope is either a string or an iterable. If string,
check if it is allowed for our access token otherwise, iterate through
the required_scopes to see which scopes are allowed
"""
# for non iterable types
if isinstance(required_scopes, six.string_types):
if token.allow_scopes(required_scopes.split()):
return [required_scopes]
return []
allowed_scopes = []
try:
for scope in required_scopes:
if token.allow_scopes(scope.split()):
allowed_scopes.append(scope)
except Exception:
raise Exception('Invalid required scope values')
else:
return allowed_scopes
2 changes: 1 addition & 1 deletion src/clist/api/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
from tastypie import fields, http
from tastypie.authentication import ApiKeyAuthentication, MultiAuthentication, SessionAuthentication
from tastypie.resources import NamespacedModelResource as ModelResource
from tastypie_oauth.authentication import OAuth2ScopedAuthentication

from clist.api.authentication import OAuth2ScopedAuthentication
from clist.api.throttle import CustomCacheThrottle


Expand Down
18 changes: 18 additions & 0 deletions src/clist/migrations/0142_resource_country_update_time.py

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit c3d2b9e

Please sign in to comment.