From a157d2db0e52c26de898f433b74f36bc39b36354 Mon Sep 17 00:00:00 2001 From: rr- Date: Sat, 2 Apr 2016 14:40:10 +0200 Subject: [PATCH] server/users: add searching prototype --- server/szurubooru/api/base_api.py | 3 +- server/szurubooru/api/user_api.py | 23 ++- server/szurubooru/app.py | 18 ++ server/szurubooru/errors.py | 3 + server/szurubooru/services/search/__init__.py | 2 + .../services/search/base_search_config.py | 98 +++++++++++ server/szurubooru/services/search/criteria.py | 23 +++ .../services/search/search_executor.py | 120 +++++++++++++ .../services/search/user_search_config.py | 45 +++++ server/szurubooru/tests/__init__.py | 0 server/szurubooru/tests/database_test_case.py | 11 ++ server/szurubooru/tests/services/__init__.py | 0 .../tests/services/search/__init__.py | 0 .../search/test_user_search_config.py | 164 ++++++++++++++++++ server/{ => szurubooru}/tests/test_util.py | 0 15 files changed, 503 insertions(+), 7 deletions(-) create mode 100644 server/szurubooru/services/search/__init__.py create mode 100644 server/szurubooru/services/search/base_search_config.py create mode 100644 server/szurubooru/services/search/criteria.py create mode 100644 server/szurubooru/services/search/search_executor.py create mode 100644 server/szurubooru/services/search/user_search_config.py create mode 100644 server/szurubooru/tests/__init__.py create mode 100644 server/szurubooru/tests/database_test_case.py create mode 100644 server/szurubooru/tests/services/__init__.py create mode 100644 server/szurubooru/tests/services/search/__init__.py create mode 100644 server/szurubooru/tests/services/search/test_user_search_config.py rename server/{ => szurubooru}/tests/test_util.py (100%) diff --git a/server/szurubooru/api/base_api.py b/server/szurubooru/api/base_api.py index 12e5ed0..f8249a6 100644 --- a/server/szurubooru/api/base_api.py +++ b/server/szurubooru/api/base_api.py @@ -5,7 +5,8 @@ import types def _bind_method(target, desired_method_name): actual_method = getattr(target, desired_method_name) def _wrapper_method(self, request, response, *args, **kwargs): - request.context.result = actual_method(request.context, *args, **kwargs) + request.context.result = actual_method( + request, request.context, *args, **kwargs) return types.MethodType(_wrapper_method, target) class BaseApi(object): diff --git a/server/szurubooru/api/user_api.py b/server/szurubooru/api/user_api.py index e971760..8c70d8e 100644 --- a/server/szurubooru/api/user_api.py +++ b/server/szurubooru/api/user_api.py @@ -1,8 +1,9 @@ ''' Users public API. ''' import sqlalchemy -from szurubooru.errors import IntegrityError, ValidationError from szurubooru.api.base_api import BaseApi +from szurubooru.errors import IntegrityError, ValidationError +from szurubooru.services.search import UserSearchConfig, SearchExecutor def _serialize_user(authenticated_user, user): ret = { @@ -23,13 +24,23 @@ class UserListApi(BaseApi): super().__init__() self._auth_service = auth_service self._user_service = user_service + self._search_executor = SearchExecutor(UserSearchConfig()) - def get(self, context): + def get(self, request, context): ''' Retrieves a list of users. ''' self._auth_service.verify_privilege(context.user, 'users:list') - return {'message': 'Searching for users'} + query = request.get_param_as_string('query') + page = request.get_param_as_int('page', 1) + count, users = self._search_executor.execute(context.session, query, page) + return { + 'query': query, + 'page': page, + 'page_size': self._search_executor.page_size, + 'total': count, + 'users': [_serialize_user(context.user, user) for user in users], + } - def post(self, context): + def post(self, request, context): ''' Creates a new user. ''' self._auth_service.verify_privilege(context.user, 'users:create') @@ -55,13 +66,13 @@ class UserDetailApi(BaseApi): self._auth_service = auth_service self._user_service = user_service - def get(self, context, user_name): + def get(self, request, context, user_name): ''' Retrieves an user. ''' self._auth_service.verify_privilege(context.user, 'users:view') user = self._user_service.get_by_name(context.session, user_name) return {'user': _serialize_user(context.user, user)} - def put(self, context, user_name): + def put(self, request, context, user_name): ''' Updates an existing user. ''' self._auth_service.verify_privilege(context.user, 'users:edit') return {'message': 'Updating user ' + user_name} diff --git a/server/szurubooru/app.py b/server/szurubooru/app.py index 533176f..03b91db 100644 --- a/server/szurubooru/app.py +++ b/server/szurubooru/app.py @@ -8,18 +8,35 @@ import szurubooru.api import szurubooru.config import szurubooru.middleware import szurubooru.services +import szurubooru.services.search import szurubooru.util from szurubooru.errors import * class _CustomRequest(falcon.Request): context_type = szurubooru.util.dotdict + def get_param_as_string(self, name, required=False, store=None, default=None): + params = self._params + if name in params: + param = params[name] + if isinstance(param, list): + param = ','.join(param) + if store is not None: + store[name] = param + return param + if not required: + return default + raise falcon.HTTPMissingParam(name) + def _on_auth_error(ex, request, response, params): raise falcon.HTTPForbidden('Authentication error', str(ex)) def _on_validation_error(ex, request, response, params): raise falcon.HTTPBadRequest('Validation error', str(ex)) +def _on_search_error(ex, request, response, params): + raise falcon.HTTPBadRequest('Search error', str(ex)) + def _on_integrity_error(ex, request, response, params): raise falcon.HTTPConflict('Integrity violation', ex.args[0]) @@ -60,6 +77,7 @@ def create_app(): app.add_error_handler(szurubooru.errors.AuthError, _on_auth_error) app.add_error_handler(szurubooru.errors.IntegrityError, _on_integrity_error) app.add_error_handler(szurubooru.errors.ValidationError, _on_validation_error) + app.add_error_handler(szurubooru.errors.SearchError, _on_search_error) app.add_route('/users/', user_list) app.add_route('/user/{user_name}', user) diff --git a/server/szurubooru/errors.py b/server/szurubooru/errors.py index 01be594..f2bec16 100644 --- a/server/szurubooru/errors.py +++ b/server/szurubooru/errors.py @@ -8,3 +8,6 @@ class IntegrityError(RuntimeError): class ValidationError(RuntimeError): ''' Validation error (e.g. trying to create user with invalid name) ''' + +class SearchError(RuntimeError): + ''' Search error (e.g. trying to use special: where it doesn't make sense) ''' diff --git a/server/szurubooru/services/search/__init__.py b/server/szurubooru/services/search/__init__.py new file mode 100644 index 0000000..c05934d --- /dev/null +++ b/server/szurubooru/services/search/__init__.py @@ -0,0 +1,2 @@ +from szurubooru.services.search.search_executor import SearchExecutor +from szurubooru.services.search.user_search_config import UserSearchConfig diff --git a/server/szurubooru/services/search/base_search_config.py b/server/szurubooru/services/search/base_search_config.py new file mode 100644 index 0000000..df2793f --- /dev/null +++ b/server/szurubooru/services/search/base_search_config.py @@ -0,0 +1,98 @@ +''' Exports BaseSearchConfig. ''' + +import sqlalchemy +from szurubooru.errors import SearchError +from szurubooru.services.search.criteria import * +from szurubooru.util import parse_time_range + +def _apply_criterion_to_column( + column, query, criterion, allow_composite=True, allow_ranged=True): + ''' Decorates SQLAlchemy filter on given column using supplied criterion. ''' + if isinstance(criterion, StringSearchCriterion): + filter = column == criterion.value + if criterion.negated: + filter = ~filter + return query.filter(filter) + elif isinstance(criterion, ArraySearchCriterion): + if not allow_composite: + raise SearchError( + 'Composite token %r is invalid in this context.' % (criterion,)) + filter = column.in_(criterion.values) + if criterion.negated: + filter = ~filter + return query.filter(filter) + elif isinstance(criterion, RangedSearchCriterion): + if not allow_ranged: + raise SearchError( + 'Ranged token %r is invalid in this context.' % (criterion,)) + filter = column.between(criterion.min_value, criterion.max_value) + if criterion.negated: + filter = ~filter + return query.filter(filter) + else: + raise RuntimeError('Invalid search type: %r.' % (criterion,)) + +def _apply_date_criterion_to_column(column, query, criterion): + ''' + Decorates SQLAlchemy filter on given column using supplied criterion. + Parses the datetime inside the criterion. + ''' + if isinstance(criterion, StringSearchCriterion): + min_date, max_date = parse_time_range(criterion.value) + filter = column.between(min_date, max_date) + if criterion.negated: + filter = ~filter + return query.filter(filter) + elif isinstance(criterion, ArraySearchCriterion): + result = query + filter = sqlalchemy.sql.false() + for value in criterion.values: + min_date, max_date = parse_time_range(value) + filter = filter | column.between(min_date, max_date) + if criterion.negated: + filter = ~filter + return query.filter(filter) + elif isinstance(criterion, RangedSearchCriterion): + assert criterion.min_value or criterion.max_value + if criterion.min_value and criterion.max_value: + min_date = parse_time_range(criterion.min_value)[0] + max_date = parse_time_range(criterion.max_value)[1] + filter = column.between(min_date, max_date) + elif criterion.min_value: + min_date = parse_time_range(criterion.min_value)[0] + filter = column >= min_date + elif criterion.max_value: + max_date = parse_time_range(criterion.max_value)[1] + filter = column <= max_date + if criterion.negated: + filter = ~filter + return query.filter(filter) + +class BaseSearchConfig(object): + def create_query(self, session): + raise NotImplementedError() + + @property + def anonymous_filter(self): + raise NotImplementedError() + + @property + def special_filters(self): + raise NotImplementedError() + + @property + def named_filters(self): + raise NotImplementedError() + + @property + def order_columns(self): + raise NotImplementedError() + + def _create_basic_filter( + self, column, allow_composite=True, allow_ranged=True): + return lambda query, criterion: _apply_criterion_to_column( + column, query, criterion, allow_composite, allow_ranged) + + def _create_date_filter(self, column): + return lambda query, criterion: _apply_date_criterion_to_column( + column, query, criterion) diff --git a/server/szurubooru/services/search/criteria.py b/server/szurubooru/services/search/criteria.py new file mode 100644 index 0000000..467cbfd --- /dev/null +++ b/server/szurubooru/services/search/criteria.py @@ -0,0 +1,23 @@ +class BaseSearchCriterion(object): + def __init__(self, original_text, negated): + self.original_text = original_text + self.negated = negated + + def __repr__(self): + return self.original_text + +class RangedSearchCriterion(BaseSearchCriterion): + def __init__(self, original_text, negated, min_value, max_value): + super().__init__(original_text, negated) + self.min_value = min_value + self.max_value = max_value + +class StringSearchCriterion(BaseSearchCriterion): + def __init__(self, original_text, negated, value): + super().__init__(original_text, negated) + self.value = value + +class ArraySearchCriterion(BaseSearchCriterion): + def __init__(self, original_text, negated, values): + super().__init__(original_text, negated) + self.values = values diff --git a/server/szurubooru/services/search/search_executor.py b/server/szurubooru/services/search/search_executor.py new file mode 100644 index 0000000..6d6a630 --- /dev/null +++ b/server/szurubooru/services/search/search_executor.py @@ -0,0 +1,120 @@ +''' Exports SearchExecutor. ''' + +import re +import sqlalchemy +from szurubooru.errors import SearchError +from szurubooru.services.search.criteria import * + +class SearchExecutor(object): + ORDER_DESC = 1 + ORDER_ASC = 2 + + ''' + Class for search parsing and execution. Handles plaintext parsing and + delegates sqlalchemy filter decoration to SearchConfig instances. + ''' + + def __init__(self, search_config): + self.page_size = 100 + self._search_config = search_config + + def execute(self, session, query_text, page): + ''' + Parse input and return tuple containing total record count and filtered + entities. + ''' + filter_query = self._prepare(session, query_text) + entities = filter_query \ + .offset((page - 1) * self.page_size).limit(self.page_size).all() + count_query = filter_query.statement \ + .with_only_columns([sqlalchemy.func.count()]).order_by(None) + count = filter_query.session.execute(count_query).scalar() + return (count, entities) + + def _prepare(self, session, query_text): + ''' Parse input and return SQLAlchemy query. ''' + query = self._search_config.create_query(session) + for token in re.split(r'\s+', (query_text or '').lower()): + if not token: + continue + negated = False + while token[0] == '-': + token = token[1:] + negated = not negated + + if ':' in token: + key, value = token.split(':', 2) + query = self._handle_key_value(query, key, value, negated) + else: + query = self._handle_anonymous( + query, self._create_criterion(token, negated)) + + return query + + def _handle_key_value(self, query, key, value, negated): + if key == 'order': + if value.count(',') == 0: + order = self.ORDER_ASC + elif value.count(',') == 1: + value, order_str = value.split(',') + if order_str == 'asc': + order = self.ORDER_ASC + elif order_str == 'desc': + order = self.ORDER_DESC + else: + raise SearchError('Unknown search direction: %r.' % order_str) + else: + raise SearchError('Too many commas in order search token.') + if negated: + if order == self.ORDER_DESC: + order = self.ORDER_ASC + else: + order = self.ORDER_DESC + return self._handle_order(query, value, order) + elif key == 'special': + return self._handle_special(query, value, negated) + else: + return self._handle_named( + query, key, self._create_criterion(value, negated)) + + def _handle_anonymous(self, query, criterion): + if not self._search_config.anonymous_filter: + raise SearchError( + 'Anonymous tokens are not valid in this context.') + return self._search_config.anonymous_filter(query, criterion) + + def _handle_named(self, query, key, criterion): + if key in self._search_config.named_filters: + return self._search_config.named_filters[key](query, criterion) + raise SearchError( + 'Unknown named token: %r. Available named tokens: %r.' % ( + key, list(self._search_config.named_filters.keys()))) + + def _handle_special(self, query, value, negated): + if value in self._search_config.special_filters: + return self._search_config.special_filters[value](query, criterion) + raise SearchError( + 'Unknown special token: %r. Available special tokens: %r.' % ( + value, list(self._search_config.special_filters.keys()))) + + def _handle_order(self, query, value, order): + if value in self._search_config.order_columns: + column = self._search_config.order_columns[value] + if order == self.ORDER_ASC: + column = column.asc() + else: + column = column.desc() + return query.order_by(column) + raise SearchError( + 'Unknown search order: %r. Available search orders: %r.' % ( + value, list(self._search_config.order_columns.keys()))) + + def _create_criterion(self, value, negated): + if '..' in value: + low, high = value.split('..') + if not low and not high: + raise SearchError('Empty ranged value') + return RangedSearchCriterion(value, negated, low, high) + if ',' in value: + return ArraySearchCriterion(value, negated, value.split(',')) + return StringSearchCriterion(value, negated, value) diff --git a/server/szurubooru/services/search/user_search_config.py b/server/szurubooru/services/search/user_search_config.py new file mode 100644 index 0000000..a9602d2 --- /dev/null +++ b/server/szurubooru/services/search/user_search_config.py @@ -0,0 +1,45 @@ +''' Exports UserSearchConfig. ''' + +from sqlalchemy.sql.expression import func +from szurubooru.errors import SearchError +from szurubooru.model import User +from szurubooru.services.search.base_search_config import BaseSearchConfig + +class UserSearchConfig(BaseSearchConfig): + ''' Executes searches related to the users. ''' + + def create_query(self, session): + return session.query(User) + + @property + def anonymous_filter(self): + return self._create_basic_filter(User.name, allow_ranged=False) + + @property + def special_filters(self): + return {} + + @property + def named_filters(self): + return { + 'name': self._create_basic_filter(User.name, allow_ranged=False), + 'creation_date': self._create_date_filter(User.creation_time), + 'creation_time': self._create_date_filter(User.creation_time), + 'last_login_date': self._create_date_filter(User.last_login_time), + 'last_login_time': self._create_date_filter(User.last_login_time), + 'login_date': self._create_date_filter(User.last_login_time), + 'login_time': self._create_date_filter(User.last_login_time), + } + + @property + def order_columns(self): + return { + 'random': func.random(), + 'name': User.name, + 'creation_date': User.creation_time, + 'creation_time': User.creation_time, + 'last_login_date': User.last_login_time, + 'last_login_time': User.last_login_time, + 'login_date': User.last_login_time, + 'login_time': User.last_login_time, + } diff --git a/server/szurubooru/tests/__init__.py b/server/szurubooru/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/szurubooru/tests/database_test_case.py b/server/szurubooru/tests/database_test_case.py new file mode 100644 index 0000000..0d89e74 --- /dev/null +++ b/server/szurubooru/tests/database_test_case.py @@ -0,0 +1,11 @@ +import unittest +import sqlalchemy +from szurubooru.model import Base + +class DatabaseTestCase(unittest.TestCase): + def setUp(self): + engine = sqlalchemy.create_engine('sqlite:///:memory:') + session_maker = sqlalchemy.orm.sessionmaker(bind=engine) + self.session = sqlalchemy.orm.scoped_session(session_maker) + Base.query = self.session.query_property() + Base.metadata.create_all(bind=engine) diff --git a/server/szurubooru/tests/services/__init__.py b/server/szurubooru/tests/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/szurubooru/tests/services/search/__init__.py b/server/szurubooru/tests/services/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/szurubooru/tests/services/search/test_user_search_config.py b/server/szurubooru/tests/services/search/test_user_search_config.py new file mode 100644 index 0000000..c3cae96 --- /dev/null +++ b/server/szurubooru/tests/services/search/test_user_search_config.py @@ -0,0 +1,164 @@ +from datetime import datetime +from szurubooru.errors import SearchError +from szurubooru.model.user import User +from szurubooru.services.search.search_executor import SearchExecutor +from szurubooru.services.search.user_search_config import UserSearchConfig +from szurubooru.tests.database_test_case import DatabaseTestCase + +class TestUserSearchExecutor(DatabaseTestCase): + def setUp(self): + super().setUp() + self.search_config = UserSearchConfig() + self.executor = SearchExecutor(self.search_config) + + def _create_user(self, name): + user = User() + user.name = name + user.password = 'dummy' + user.password_salt = 'dummy' + user.password_hash = 'dummy' + user.email = 'dummy' + user.access_rank = 'dummy' + user.creation_time = datetime.now() + user.avatar_style = User.AVATAR_GRAVATAR + return user + + def _test(self, query, page, expected_count, expected_user_names): + count, users = self.executor.execute(self.session, query, page) + self.assertEqual(count, expected_count) + self.assertEqual([u.name for u in users], expected_user_names) + + def test_filter_by_creation_time(self): + user1 = self._create_user('u1') + user2 = self._create_user('u2') + user1.creation_time = datetime(2014, 1, 1) + user2.creation_time = datetime(2015, 1, 1) + self.session.add_all([user1, user2]) + for alias in ['creation_time', 'creation_date']: + self._test('%s:2014' % alias, 1, 1, ['u1']) + + def test_filter_by_negated_creation_time(self): + user1 = self._create_user('u1') + user2 = self._create_user('u2') + user1.creation_time = datetime(2014, 1, 1) + user2.creation_time = datetime(2015, 1, 1) + self.session.add_all([user1, user2]) + for alias in ['creation_time', 'creation_date']: + self._test('-%s:2014' % alias, 1, 1, ['u2']) + + def test_filter_by_ranged_creation_time(self): + user1 = self._create_user('u1') + user2 = self._create_user('u2') + user3 = self._create_user('u3') + user1.creation_time = datetime(2014, 1, 1) + user2.creation_time = datetime(2014, 6, 1) + user3.creation_time = datetime(2015, 1, 1) + self.session.add_all([user1, user2, user3]) + for alias in ['creation_time', 'creation_date']: + self._test('%s:2014..2014-06' % alias, 1, 2, ['u1', 'u2']) + self._test('%s:2014-06..2015-01-01' % alias, 1, 2, ['u2', 'u3']) + self._test('%s:2014-06..' % alias, 1, 2, ['u2', 'u3']) + self._test('%s:..2014-06' % alias, 1, 2, ['u1', 'u2']) + self.assertRaises( + SearchError, self.executor.execute, self.session, '%s:..', 1) + + def test_filter_by_negated_ranged_creation_time(self): + user1 = self._create_user('u1') + user2 = self._create_user('u2') + user3 = self._create_user('u3') + user1.creation_time = datetime(2014, 1, 1) + user2.creation_time = datetime(2014, 6, 1) + user3.creation_time = datetime(2015, 1, 1) + self.session.add_all([user1, user2, user3]) + for alias in ['creation_time', 'creation_date']: + self._test('-%s:2014..2014-06' % alias, 1, 1, ['u3']) + self._test('-%s:2014-06..2015-01-01' % alias, 1, 1, ['u1']) + + def test_filter_by_composite_creation_time(self): + user1 = self._create_user('u1') + user2 = self._create_user('u2') + user3 = self._create_user('u3') + user1.creation_time = datetime(2014, 1, 1) + user2.creation_time = datetime(2014, 6, 1) + user3.creation_time = datetime(2015, 1, 1) + self.session.add_all([user1, user2, user3]) + for alias in ['creation_time', 'creation_date']: + self._test('%s:2014-01,2015' % alias, 1, 2, ['u1', 'u3']) + + def test_filter_by_negated_composite_creation_time(self): + user1 = self._create_user('u1') + user2 = self._create_user('u2') + user3 = self._create_user('u3') + user1.creation_time = datetime(2014, 1, 1) + user2.creation_time = datetime(2014, 6, 1) + user3.creation_time = datetime(2015, 1, 1) + self.session.add_all([user1, user2, user3]) + for alias in ['creation_time', 'creation_date']: + self._test('-%s:2014-01,2015' % alias, 1, 1, ['u2']) + + def test_filter_by_name(self): + self.session.add(self._create_user('u1')) + self.session.add(self._create_user('u2')) + self._test('name:u1', 1, 1, ['u1']) + self._test('name:u2', 1, 1, ['u2']) + + def test_filter_by_negated_name(self): + self.session.add(self._create_user('u1')) + self.session.add(self._create_user('u2')) + self._test('-name:u1', 1, 1, ['u2']) + self._test('-name:u2', 1, 1, ['u1']) + + def test_filter_by_composite_name(self): + self.session.add(self._create_user('u1')) + self.session.add(self._create_user('u2')) + self.session.add(self._create_user('u3')) + self._test('name:u1,u2', 1, 2, ['u1', 'u2']) + + def test_filter_by_ranged_name(self): + self.assertRaises( + SearchError, self.executor.execute, self.session, 'name:u1..u2', 1) + + def test_paging(self): + self.executor.page_size = 1 + self.session.add(self._create_user('u1')) + self.session.add(self._create_user('u2')) + self._test('', 1, 2, ['u1']) + self._test('', 2, 2, ['u2']) + + def test_order_by_name(self): + self.session.add(self._create_user('u1')) + self.session.add(self._create_user('u2')) + self._test('order:name', 1, 2, ['u1', 'u2']) + self._test('-order:name', 1, 2, ['u2', 'u1']) + self._test('order:name,asc', 1, 2, ['u1', 'u2']) + self._test('order:name,desc', 1, 2, ['u2', 'u1']) + self._test('-order:name,asc', 1, 2, ['u2', 'u1']) + self._test('-order:name,desc', 1, 2, ['u1', 'u2']) + + def test_anonymous(self): + self.session.add(self._create_user('u1')) + self.session.add(self._create_user('u2')) + self._test('u1', 1, 1, ['u1']) + self._test('u2', 1, 1, ['u2']) + + def test_negated_anonymous(self): + self.session.add(self._create_user('u1')) + self.session.add(self._create_user('u2')) + self._test('-u1', 1, 1, ['u2']) + self._test('-u2', 1, 1, ['u1']) + + def test_combining(self): + user1 = self._create_user('u1') + user2 = self._create_user('u2') + user3 = self._create_user('u3') + user1.creation_time = datetime(2014, 1, 1) + user2.creation_time = datetime(2014, 6, 1) + user3.creation_time = datetime(2015, 1, 1) + self.session.add_all([user1, user2, user3]) + self._test('creation_time:2014 u1', 1, 1, ['u1']) + self._test('creation_time:2014 u2', 1, 1, ['u2']) + self._test('creation_time:2016 u2', 1, 0, []) + + def test_special(self): + self.assertRaises( + SearchError, self.executor.execute, self.session, 'special:-', 1) diff --git a/server/tests/test_util.py b/server/szurubooru/tests/test_util.py similarity index 100% rename from server/tests/test_util.py rename to server/szurubooru/tests/test_util.py