Accords-Library-Archive
/
gallery.accords-library.com
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

server/tags: export also tag categories

This commit is contained in:
rr- 2016-04-19 15:45:12 +02:00
parent 884747bbbd
commit fe56e376f6
7 changed files with 122 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
API.md
View File

@ -14,7 +14,7 @@
2. [API reference](#api-reference)
- Tag categories
- [Listing tag categories](#listing-tags-categories)
- [Listing tag categories](#listing-tag-categories)
- [Creating tag category](#creating-tag-category)
- [Updating tag category](#updating-tag-category)
- [Getting tag category](#getting-tag-category)
@ -117,6 +117,12 @@ data.
Lists all tag categories. Doesn't support paging.
**Note**: independently, the server exports current tag category list
snapshots to the data directory under `tags.json` name. Its purpose is to
reduce the trips frontend needs to make when doing autocompletion, and ease
caching. The data directory and its URL are controlled with `data_dir` and
`data_url` variables in server's configuration.
## Creating tag category
- **Request**
@ -231,6 +237,7 @@ data.
- the tag category does not exist
- the tag category is used by some tags
- the tag category is the last tag category available
- privileges are too low
- **Description**
@ -352,7 +359,7 @@ data.
- **Errors**
- any name is used by an existing tag (names are case insensitive)
- any name, implication or suggestion has invalid name
- any name, implication or is invalid
- category is invalid
- no name was specified
- implications or suggestions contain any item from names (e.g. there's a
@ -411,12 +418,12 @@ data.
Updates an existing tag using specified parameters. Names, suggestions and
implications must match `tag_name_regex` from server's configuration.
Category must be one of `tag_categories` from server's configuration.
If specified implied tags or suggested tags do not exist yet, they will
be automatically created. Tags created automatically have no implications,
no suggestions, one name and their category is set to the first item of
`tag_categories` from server's configuration. All fields are optional -
update concerns only provided fields.
Category must exist and is the same as `name` field within
[`<tag-category>` resource](#tag-category). If specified implied tags or
suggested tags do not exist yet, they will be automatically created. Tags
created automatically have no implications, no suggestions, one name and
their category is set to the first tag category found. All fields are
optional - update concerns only provided fields.
## Getting tag

6
config.yaml.dist
View File

@ -99,7 +99,8 @@ privileges:
'tags:edit:category': power_user
'tags:edit:implications': power_user
'tags:edit:suggestions': power_user
'tags:list': regular_user
'tags:list': regular_user # note: will be available as data_url/tags.json anyway
'tags:view': anonymous
'tags:masstag': power_user
'tags:merge': mod
'tags:delete': mod
@ -107,7 +108,8 @@ privileges:
'tag_categories:create': mod
'tag_categories:edit:name': mod
'tag_categories:edit:color': mod
'tag_categories:list': anonymous
'tag_categories:list': anonymous # note: will be available as data_url/tags.json anyway
'tag_categories:view': anonymous
'tag_categories:delete': mod
'comments:create': regular_user

10
server/szurubooru/db/tag.py
View File

@ -47,18 +47,20 @@ class Tag(Base):
creation_time = Column('creation_time', DateTime, nullable=False)
last_edit_time = Column('last_edit_time', DateTime)
category = relationship('TagCategory')
names = relationship('TagName', cascade='all, delete-orphan')
category = relationship('TagCategory', lazy='joined')
names = relationship('TagName', cascade='all, delete-orphan', lazy='joined')
suggestions = relationship(
'Tag',
secondary='tag_suggestion',
primaryjoin=tag_id == TagSuggestion.parent_id,
secondaryjoin=tag_id == TagSuggestion.child_id)
secondaryjoin=tag_id == TagSuggestion.child_id,
lazy='joined')
implications = relationship(
'Tag',
secondary='tag_implication',
primaryjoin=tag_id == TagImplication.parent_id,
secondaryjoin=tag_id == TagImplication.child_id)
secondaryjoin=tag_id == TagImplication.child_id,
lazy='joined')
post_count = column_property(
select([func.count('Post.post_id')]) \

10
server/szurubooru/search/search_executor.py
View File

@ -21,13 +21,17 @@ class SearchExecutor(object):
entities = filter_query \
.offset((page - 1) * page_size).limit(page_size).all()
count_query = filter_query.statement \
.with_only_columns([sqlalchemy.func.count()]).order_by(None)
count = filter_query.session.execute(count_query).scalar()
.with_only_columns([sqlalchemy.func.count()]) \
.order_by(None)
count = filter_query \
.session.execute(count_query) \
.scalar()
return (count, entities)
def _prepare(self, query_text):
''' Parse input and return SQLAlchemy query. '''
query = self._search_config.create_query()
query = self._search_config.create_query() \
.options(sqlalchemy.orm.lazyload('*'))
for token in re.split(r'\s+', (query_text or '').lower()):
if not token:
continue

60
server/szurubooru/tests/api/test_tag_export.py
View File

@ -4,17 +4,27 @@ import json
from szurubooru import config, db
from szurubooru.util import tags
def test_export(tmpdir, session, config_injector, tag_factory):
def test_export(
tmpdir,
query_counter,
session,
config_injector,
tag_factory,
tag_category_factory):
config_injector({
'data_dir': str(tmpdir)
})
sug1 = tag_factory(names=['sug1'])
sug2 = tag_factory(names=['sug2'])
imp1 = tag_factory(names=['imp1'])
imp2 = tag_factory(names=['imp2'])
tag = tag_factory(names=['alias1', 'alias2'])
cat1 = tag_category_factory(name='cat1', color='black')
cat2 = tag_category_factory(name='cat2', color='white')
session.add_all([cat1, cat2])
session.flush()
sug1 = tag_factory(names=['sug1'], category=cat1)
sug2 = tag_factory(names=['sug2'], category=cat1)
imp1 = tag_factory(names=['imp1'], category=cat1)
imp2 = tag_factory(names=['imp2'], category=cat1)
tag = tag_factory(names=['alias1', 'alias2'], category=cat2)
tag.post_count = 1
session.add_all([tag, sug1, sug2, imp1, imp2])
session.add_all([tag, sug1, sug2, imp1, imp2, cat1, cat2])
session.flush()
session.add_all([
db.TagSuggestion(tag.tag_id, sug1.tag_id),
@ -24,19 +34,29 @@ def test_export(tmpdir, session, config_injector, tag_factory):
])
session.flush()
tags.export_to_json()
with query_counter:
tags.export_to_json()
assert len(query_counter.statements) == 2
export_path = os.path.join(config.config['data_dir'], 'tags.json')
assert os.path.exists(export_path)
with open(export_path, 'r') as handle:
assert json.loads(handle.read()) == [
{
'names': ['alias1', 'alias2'],
'usages': 1,
'suggestions': ['sug1', 'sug2'],
'implications': ['imp1', 'imp2'],
},
{'names': ['sug1'], 'usages': 0},
{'names': ['sug2'], 'usages': 0},
{'names': ['imp1'], 'usages': 0},
{'names': ['imp2'], 'usages': 0},
]
assert json.loads(handle.read()) == {
'tags': [
{
'names': ['alias1', 'alias2'],
'usages': 1,
'category': 'cat2',
'suggestions': ['sug1', 'sug2'],
'implications': ['imp1', 'imp2'],
},
{'names': ['sug1'], 'usages': 0, 'category': 'cat1'},
{'names': ['sug2'], 'usages': 0, 'category': 'cat1'},
{'names': ['imp1'], 'usages': 0, 'category': 'cat1'},
{'names': ['imp2'], 'usages': 0, 'category': 'cat1'},
],
'categories': [
{'name': 'cat1', 'color': 'black'},
{'name': 'cat2', 'color': 'white'},
]
}

32
server/szurubooru/tests/conftest.py
View File

@ -7,6 +7,32 @@ import sqlalchemy
from szurubooru import api, config, db
from szurubooru.util import misc
class QueryCounter(object):
def __init__(self):
self._statements = []
def __enter__(self):
self._statements = []
def __exit__(self, *args, **kwargs):
self._statements = []
def create_before_cursor_execute(self):
def before_cursor_execute(
_conn, _cursor, statement, _parameters, _context, _executemany):
self._statements.append(statement)
return before_cursor_execute
@property
def statements(self):
return self._statements
_query_counter = QueryCounter()
@pytest.fixture
def query_counter():
return _query_counter
def get_unique_name():
return str(uuid.uuid4())
@ -21,11 +47,15 @@ def fake_datetime():
return injector
@pytest.yield_fixture
def session(autoload=True):
def session(query_counter, autoload=True):
import logging
logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
engine = sqlalchemy.create_engine('sqlite:///:memory:')
sqlalchemy.event.listen(
engine,
'before_cursor_execute',
query_counter.create_before_cursor_execute())
session_maker = sqlalchemy.orm.sessionmaker(bind=engine)
session = sqlalchemy.orm.scoped_session(session_maker)
db.Base.query = session.query_property()

23
server/szurubooru/util/tags.py
View File

@ -28,11 +28,21 @@ def _check_name_intersection(names1, names2):
return len(set(_lower_list(names1)).intersection(_lower_list(names2))) > 0
def export_to_json():
output = []
for tag in db.session().query(db.Tag).all():
output = {
'tags': [],
'categories': [],
}
all_tags = db.session() \
.query(db.Tag) \
.options(
sqlalchemy.orm.joinedload('suggestions'),
sqlalchemy.orm.joinedload('implications')) \
.all()
for tag in all_tags:
item = {
'names': [tag_name.name for tag_name in tag.names],
'usages': tag.post_count
'usages': tag.post_count,
'category': tag.category.name,
}
if len(tag.suggestions):
item['suggestions'] = \
@ -40,7 +50,12 @@ def export_to_json():
if len(tag.implications):
item['implications'] = \
[rel.names[0].name for rel in tag.implications]
output.append(item)
output['tags'].append(item)
for category in tag_categories.get_all_categories():
output['categories'].append({
'name': category.name,
'color': category.color,
})
export_path = os.path.join(config.config['data_dir'], 'tags.json')
with open(export_path, 'w') as handle:
handle.write(json.dumps(output, separators=(',', ':')))