Here are the examples of the python api sqlalchemy.orm.joinedload_all taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
44 Examples
3
Example 1
Project: quicktill Source File: views.py
@tillweb_view
def transaction(request,info,session,transid):
try:
t=session.query(Transaction).\
filter_by(id=int(transid)).\
options(subqueryload_all('payments')).\
options(joinedload_all('lines.stockref.stockitem.stocktype')).\
options(joinedload('lines.user')).\
options(undefer('total')).\
one()
except NoResultFound:
raise Http404
return ('transaction.html',{'transaction':t,})
3
Example 2
Project: quicktill Source File: views.py
@tillweb_view
def transline(request,info,session,translineid):
try:
tl=session.query(Transline).\
filter_by(id=int(translineid)).\
options(joinedload_all('stockref.stockitem.stocktype')).\
options(joinedload('user')).\
one()
except NoResultFound:
raise Http404
return ('transline.html',{'tl':tl,})
3
Example 3
@tillweb_view
def delivery(request,info,session,deliveryid):
try:
d=session.query(Delivery).\
filter_by(id=int(deliveryid)).\
options(joinedload_all('items.stocktype.unit')).\
options(joinedload_all('items.stockline')).\
options(undefer_group('qtys')).\
one()
except NoResultFound:
raise Http404
return ('delivery.html',{'delivery':d,})
3
Example 4
Project: quicktill Source File: views.py
@tillweb_view
def stocksearch(request,info,session):
form=StockForm(request.GET)
result=[]
q=session.query(StockItem).join(StockType).order_by(StockItem.id).\
options(joinedload_all('stocktype.unit')).\
options(joinedload('stockline')).\
options(undefer_group('qtys'))
if form.is_valid():
if form.is_filled_in():
q=form.filter(q)
if not form.cleaned_data['include_finished']:
q=q.filter(StockItem.finished==None)
result=q.all()
return ('stocksearch.html',{'form':form,'stocklist':result})
3
Example 5
@tillweb_view
def stock(request,info,session,stockid):
try:
s=session.query(StockItem).\
filter_by(id=int(stockid)).\
options(joinedload_all('stocktype.department')).\
options(joinedload_all('stocktype.stockline_log.stockline')).\
options(joinedload_all('delivery.supplier')).\
options(joinedload_all('stockunit.unit')).\
options(joinedload_all('annotations.type')).\
options(subqueryload_all('out.transline.transaction')).\
options(undefer_group('qtys')).\
one()
except NoResultFound:
raise Http404
return ('stock.html',{'stock':s,})
3
Example 6
Project: quicktill Source File: views.py
@tillweb_view
def stockline(request,info,session,stocklineid):
try:
s=session.query(StockLine).\
filter_by(id=int(stocklineid)).\
options(joinedload_all('stockonsale.stocktype.unit')).\
options(undefer_group('qtys')).\
one()
except NoResultFound:
raise Http404
return ('stockline.html',{'stockline':s,})
3
Example 7
Project: quicktill Source File: views.py
@tillweb_view
def department(request,info,session,departmentid):
d=session.query(Department).get(int(departmentid))
if d is None: raise Http404
include_finished=request.GET.get("show_finished","off")=="on"
items=session.query(StockItem).\
join(StockType).\
filter(StockType.department==d).\
order_by(desc(StockItem.id)).\
options(joinedload_all('stocktype.unit')).\
options(undefer_group('qtys')).\
options(joinedload('stockline')).\
options(joinedload('finishcode'))
if not include_finished:
items=items.filter(StockItem.finished==None)
items=items.all()
return ('department.html',{'department':d,'items':items,
'include_finished':include_finished})
3
Example 8
Project: baruwa2 Source File: base.py
def _get_domain(self, domainid):
"utility to return domain"
try:
cachekey = 'domain-%s' % domainid
qry = Session.query(Domain).filter(Domain.id == domainid)\
.options(joinedload_all(Domain.servers),
joinedload_all(Domain.aliases),
joinedload_all(Domain.authservers))\
.options(FromCache('sql_cache_med', cachekey))
if self.invalidate:
qry.invalidate()
domain = qry.one()
except NoResultFound:
domain = None
return domain
3
Example 9
Project: reggata Source File: commands.py
def _execute(self, uow):
self._session = uow.session
item = self._session.query(db.Item)\
.options(joinedload_all('data_ref'))\
.options(joinedload_all('item_tags.tag'))\
.options(joinedload_all('item_fields.field'))\
.get(self.__itemId)
if item is None:
raise err.NotFoundError()
self._session.expunge(item)
return item
3
Example 10
Project: sqlalchemy Source File: test_of_type.py
def test_twolevel_joinedload_wsubclass(self):
ParentThing, DataContainer, Job, SubJob = \
self.classes.ParentThing,\
self.classes.DataContainer,\
self.classes.Job,\
self.classes.SubJob
s = Session(testing.db)
q = s.query(ParentThing).\
options(
joinedload_all(
ParentThing.container,
DataContainer.jobs.of_type(SubJob)
))
def go():
eq_(
q.all(),
self._fixture()
)
self.assert_sql_count(testing.db, go, 5)
3
Example 11
Project: sqlalchemy Source File: test_options.py
def test_option_against_nonexistent_twolevel_all(self):
Item = self.classes.Item
self._assert_eager_with_entity_exception(
[Item],
(joinedload_all("keywords.foo"), ),
r"Can't find property named 'foo' on the mapped entity "
r"Mapper\|Keyword\|keywords in this Query."
)
3
Example 12
Project: sqlalchemy Source File: test_options.py
@testing.fails_if(lambda: True,
"PropertyOption doesn't yet check for relation/column on end result")
def test_option_against_non_relation_basestring(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword, Item],
(joinedload_all("keywords"), ),
r"Attribute 'keywords' of entity 'Mapper\|Keyword\|keywords' "
"does not refer to a mapped entity"
)
3
Example 13
Project: sqlalchemy Source File: test_options.py
@testing.fails_if(lambda: True,
"PropertyOption doesn't yet check for relation/column on end result")
def test_option_against_multi_non_relation_basestring(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword, Item],
(joinedload_all("keywords"), ),
r"Attribute 'keywords' of entity 'Mapper\|Keyword\|keywords' "
"does not refer to a mapped entity"
)
3
Example 14
Project: sqlalchemy Source File: test_options.py
def test_option_against_wrong_entity_type_basestring(self):
Item = self.classes.Item
self._assert_eager_with_entity_exception(
[Item],
(joinedload_all("id", "keywords"), ),
r"Attribute 'id' of entity 'Mapper\|Item\|items' does not "
r"refer to a mapped entity"
)
3
Example 15
Project: sqlalchemy Source File: test_options.py
def test_option_against_multi_non_relation_twolevel_basestring(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword, Item],
(joinedload_all("id", "keywords"), ),
r"Attribute 'id' of entity 'Mapper\|Keyword\|keywords' "
"does not refer to a mapped entity"
)
3
Example 16
Project: sqlalchemy Source File: test_options.py
def test_option_against_multi_nonexistent_basestring(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword, Item],
(joinedload_all("description"), ),
r"Can't find property named 'description' on the mapped "
r"entity Mapper\|Keyword\|keywords in this Query."
)
3
Example 17
Project: sqlalchemy Source File: test_options.py
def test_option_against_multi_no_entities_basestring(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword.id, Item.id],
(joinedload_all("keywords"), ),
r"Query has only expression-based entities - can't find property "
"named 'keywords'."
)
3
Example 18
Project: sqlalchemy Source File: test_options.py
def test_option_against_wrong_multi_entity_type_attr_one(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword, Item],
(joinedload_all(Keyword.id, Item.keywords), ),
r"Attribute 'id' of entity 'Mapper\|Keyword\|keywords' "
"does not refer to a mapped entity"
)
3
Example 19
Project: sqlalchemy Source File: test_options.py
def test_option_against_wrong_multi_entity_type_attr_two(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword, Item],
(joinedload_all(Keyword.keywords, Item.keywords), ),
r"Attribute 'keywords' of entity 'Mapper\|Keyword\|keywords' "
"does not refer to a mapped entity"
)
3
Example 20
Project: sqlalchemy Source File: test_options.py
def test_option_against_wrong_multi_entity_type_attr_three(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Keyword.id, Item.id],
(joinedload_all(Keyword.keywords, Item.keywords), ),
r"Query has only expression-based entities - "
"can't find property named 'keywords'."
)
3
Example 21
Project: sqlalchemy Source File: test_options.py
def test_wrong_type_in_option(self):
Item = self.classes.Item
Keyword = self.classes.Keyword
self._assert_eager_with_entity_exception(
[Item],
(joinedload_all(Keyword), ),
r"mapper option expects string key or list of attributes"
)
3
Example 22
Project: sqlalchemy Source File: test_options.py
def test_non_contiguous_all_option(self):
User = self.classes.User
self._assert_eager_with_entity_exception(
[User],
(joinedload_all(User.addresses, User.orders), ),
r"Attribute 'User.orders' does not link "
"from element 'Mapper|Address|addresses'"
)
3
Example 23
Project: sqlalchemy Source File: test_options.py
def test_non_contiguous_all_option_of_type(self):
User = self.classes.User
Order = self.classes.Order
self._assert_eager_with_entity_exception(
[User],
(joinedload_all(User.addresses, User.orders.of_type(Order)), ),
r"Attribute 'User.orders' does not link "
"from element 'Mapper|Address|addresses'"
)
3
Example 24
def main(args): # pragma: no cover
global MAX_IDENTIFIER_PK
stats = Counter()
with transaction.manager:
MAX_IDENTIFIER_PK = DBSession.query(
Identifier.pk).order_by(desc(Identifier.pk)).first()[0]
gl_names = glottolog_names()
for l in DBSession.query(Languoid).options(joinedload_all(
Language.languageidentifier, LanguageIdentifier.identifier
)):
stats.update(create_name(gl_names, l))
args.log.info('%s' % stats)
3
Example 25
Project: glottolog3 Source File: __init__.py
def refined_query(self, query, model, req):
if model == Language:
query = query.options(
joinedload(models.Languoid.family),
joinedload(models.Languoid.children),
joinedload_all(
Language.valuesets, ValueSet.references, ValueSetReference.source)
)
return query
3
Example 26
Project: ggrc-core Source File: attribute_query.py
def process_eager_loading(self, value):
paths = value.split(',')
options = []
for path in paths:
segments = path.split('.')
real_segments = []
current_model = self.model
for segment in segments:
real_segment, current_model = self.resolve_path_segment(
segment, current_model)
if current_model is not None:
real_segments.append(real_segment)
realized_path = '.'.join(real_segments)
options.append(joinedload_all(realized_path))
return options
0
Example 27
Project: quicktill Source File: managestock.py
@user.permission_required('stock-history', 'List finished stock')
def stockhistory(dept=None):
log.info("Stock history")
sq = td.s.query(StockItem)\
.join(StockItem.stocktype)\
.filter(StockItem.finished != None)\
.options(undefer(StockItem.remaining))\
.options(joinedload_all('stocktype.unit'))\
.order_by(StockItem.id.desc())
if dept:
sq = sq.filter(StockType.dept_id == dept)
sinfo = sq.all()
f = ui.tableformatter(' r l l ')
sl = [(f(x.id, x.stocktype.format(), x.remaining_units),
stock.stockinfo_popup, (x.id,)) for x in sinfo]
title = "Stock History" if dept is None \
else "Stock History department {}".format(dept)
ui.menu(sl, title=title, blurb="Select a stock item and press "
"Cash/Enter for more information. The number of units remaining "
"when the stock was finished is shown.", dismiss_on_select=False)
0
Example 28
Project: quicktill Source File: views.py
@tillweb_view
def pubroot(request,info,session):
date=datetime.date.today()
# If it's the early hours of the morning, it's more useful for us
# to consider it still to be yesterday.
if datetime.datetime.now().hour<4: date=date-datetime.timedelta(1)
thisweek_start=date-datetime.timedelta(date.weekday())
thisweek_end=thisweek_start+datetime.timedelta(6)
lastweek_start=thisweek_start-datetime.timedelta(7)
lastweek_end=thisweek_end-datetime.timedelta(7)
weekbefore_start=lastweek_start-datetime.timedelta(7)
weekbefore_end=lastweek_end-datetime.timedelta(7)
weeks=[("Current week",thisweek_start,thisweek_end,
business_totals(session,thisweek_start,thisweek_end)),
("Last week",lastweek_start,lastweek_end,
business_totals(session,lastweek_start,lastweek_end)),
("The week before last",weekbefore_start,weekbefore_end,
business_totals(session,weekbefore_start,weekbefore_end))]
currentsession=Session.current(session)
barsummary=session.query(StockLine).\
filter(StockLine.location=="Bar").\
order_by(StockLine.dept_id,StockLine.name).\
options(joinedload_all('stockonsale.stocktype.unit')).\
options(undefer_group('qtys')).\
all()
stillage=session.query(StockAnnotation).\
join(StockItem).\
outerjoin(StockLine).\
filter(tuple_(StockAnnotation.text,StockAnnotation.time).in_(
select([StockAnnotation.text,func.max(StockAnnotation.time)],
StockAnnotation.atype=='location').\
group_by(StockAnnotation.text))).\
filter(StockItem.finished==None).\
order_by(StockLine.name!=null(),StockAnnotation.time).\
options(joinedload_all('stockitem.stocktype.unit')).\
options(joinedload_all('stockitem.stockline')).\
options(undefer_group('qtys')).\
all()
return ('index.html',
{'currentsession':currentsession,
'barsummary':barsummary,
'stillage':stillage,
'weeks':weeks,
})
0
Example 29
Project: quicktill Source File: views.py
@tillweb_view
def sessiondept(request,info,session,sessionid,dept):
try:
s=session.query(Session).filter_by(id=int(sessionid)).one()
except NoResultFound:
raise Http404
try:
dept=session.query(Department).filter_by(id=int(dept)).one()
except NoResultFound:
raise Http404
nextsession=session.query(Session).\
filter(Session.id>s.id).\
order_by(Session.id).\
first()
nextlink=info['base']+nextsession.tillweb_url+"dept{}/".format(dept.id) \
if nextsession else None
prevsession=session.query(Session).\
filter(Session.id<s.id).\
order_by(desc(Session.id)).\
first()
prevlink=info['base']+prevsession.tillweb_url+"dept{}/".format(dept.id) \
if prevsession else None
translines=session.query(Transline).\
join(Transaction).\
options(joinedload('transaction')).\
options(joinedload('user')).\
options(joinedload_all('stockref.stockitem.stocktype.unit')).\
filter(Transaction.sessionid==s.id).\
filter(Transline.dept_id==dept.id).\
order_by(Transline.id).\
all()
return ('sessiondept.html',{'session':s,'department':dept,
'translines':translines,
'nextlink':nextlink,'prevlink':prevlink})
0
Example 30
@tillweb_view
def user(request,info,session,userid):
try:
u=session.query(User).\
options(joinedload('permissions')).\
options(joinedload('tokens')).\
get(int(userid))
except NoResultFound:
raise Http404
sales=session.query(Transline).filter(Transline.user==u).\
options(joinedload('transaction')).\
options(joinedload_all('stockref.stockitem.stocktype.unit')).\
order_by(desc(Transline.time))[:50]
payments=session.query(Payment).filter(Payment.user==u).\
options(joinedload('transaction')).\
options(joinedload('paytype')).\
order_by(desc(Payment.time))[:50]
annotations=session.query(StockAnnotation).\
options(joinedload_all('stockitem.stocktype')).\
options(joinedload('type')).\
filter(StockAnnotation.user==u).\
order_by(desc(StockAnnotation.time))[:50]
return ('user.html',{'tuser':u,'sales':sales,'payments':payments,
'annotations':annotations})
0
Example 31
Project: reggata Source File: commands.py
def __getFileInfo(self, path):
try:
data_ref = self._session.query(db.DataRef)\
.filter(db.DataRef.url_raw==hlp.to_db_format(path))\
.options(joinedload_all("items"))\
.options(joinedload_all("items.item_tags.tag"))\
.options(joinedload_all("items.item_fields.field"))\
.one()
self._session.expunge(data_ref)
finfo = FileInfo(data_ref.url, objType=FileInfo.FILE, status=FileInfo.STORED)
for item in data_ref.items:
finfo.itemIds.append(item.id)
for item_tag in item.item_tags:
finfo.tags.append(item_tag.tag.name)
for item_field in item.item_fields:
finfo.fields.append((item_field.field.name, item_field.field_value))
return finfo
except NoResultFound:
return FileInfo(self.__relPath, status=FileInfo.UNTRACKED)
0
Example 32
Project: sqlalchemy Source File: test_assorted_eager.py
def test_joinedload_on_path(self):
Entry, Account, Transaction = (self.classes.Entry,
self.classes.Account,
self.classes.Transaction)
session = create_session()
tx1 = Transaction(name='tx1')
tx2 = Transaction(name='tx2')
acc1 = Account(name='acc1')
ent11 = Entry(name='ent11', account=acc1, transaction=tx1)
ent12 = Entry(name='ent12', account=acc1, transaction=tx2)
acc2 = Account(name='acc2')
ent21 = Entry(name='ent21', account=acc2, transaction=tx1)
ent22 = Entry(name='ent22', account=acc2, transaction=tx2)
session.add(acc1)
session.flush()
session.expunge_all()
def go():
# load just the first Account. eager loading will actually load
# all objects saved thus far, but will not eagerly load the
# "accounts" off the immediate "entries"; only the "accounts" off
# the entries->transaction->entries
acc = (session.query(Account).
options(sa.orm.joinedload_all('entries.transaction.entries.account')).
order_by(Account.account_id)).first()
# no sql occurs
eq_(acc.name, 'acc1')
eq_(acc.entries[0].transaction.entries[0].account.name, 'acc1')
eq_(acc.entries[0].transaction.entries[1].account.name, 'acc2')
# lazyload triggers but no sql occurs because many-to-one uses
# cached query.get()
for e in acc.entries:
assert e.account is acc
self.assert_sql_count(testing.db, go, 1)
0
Example 33
def query(self, req):
return req.db.query(Language).options(joinedload_all(
Language.languageidentifier, LanguageIdentifier.identifier))\
.order_by(Language.pk)
0
Example 34
Project: glottolog3 Source File: langdocstatus.py
def main(args): # pragma: no cover
# we merge information about extinct languages from unesco and Harald.
extinct = dict(list(dsv.reader(args.data_file('extinct.tab'))))
with transaction.manager:
query = language_query().options(
joinedload_all(Language.valuesets, ValueSet.values))
# loop over active, established languages with geo-coords
for l in page_query(query, n=100, verbose=True):
# let's collect the relevant sources in a way that allows computation of med.
# Note: we limit refs to the ones without computerized assignments.
sources = DBSession.query(Ref).join(LanguageSource)\
.filter(LanguageSource.language_pk == l.pk) \
.filter(Ref.ca_doctype_trigger == None)\
.filter(Ref.ca_language_trigger == None)\
.options(joinedload(Ref.doctypes))
sources = sorted(map(Source, sources))
# keep the overall med
# note: this source may not be included in the potential meds computed below,
# e.g. because it may not have a year.
med = sources[0].__json__() if sources else None
# now we have to compute meds respecting a cut-off year.
# to do so, we collect eligible sources per year and then
# take the med of this collection.
potential_meds = []
# we only have to loop over publication years within all sources, because
# only in these years something better might have come along.
for year in set(s.year for s in sources if s.year):
# let's see if something better was published!
eligible = [s for s in sources if s.year and s.year <= year]
if eligible:
potential_meds.append(sorted(eligible)[0])
# we store the precomputed sources information as jsondata:
l.update_jsondata(
endangerment='Extinct' if l.hid in extinct else l.endangerment,
med=med,
sources=[s.__json__() for s in
sorted(set(potential_meds), key=lambda s: -s.year)])
0
Example 35
Project: glottolog3 Source File: languages_and_dialects_geo_download.py
def main(args): # pragma: no cover
rows = [
['glottocode', 'name', 'isocodes', 'level', 'macroarea', 'latitude', 'longitude']]
version_pattern = re.compile('glottolog\-(?P<version>[0-9]+\.[0-9]+)')
with transaction.manager:
# loop over languages and dialects
for l in DBSession.query(Languoid)\
.filter(Language.active)\
.filter(or_(
Languoid.level == LanguoidLevel.dialect,
and_(
Languoid.level == LanguoidLevel.language,
Languoid.status == LanguoidStatus.established)))\
.options(
joinedload(Languoid.macroareas),
joinedload_all(
Language.languageidentifier, LanguageIdentifier.identifier))\
.order_by(Language.name):
rows.append([
l.id,
l.name,
' '.join(i.name for i in l.get_identifier_objs(IdentifierType.iso)),
l.level,
l.macroareas[0].name if l.macroareas else '',
l.latitude if l.latitude is not None else '',
l.longitude if l.longitude is not None else ''])
outdir = args.module_dir.joinpath('static', 'download')
match = version_pattern.search(args.config_uri)
if match:
outdir = outdir.joinpath(match.group('version'))
if not outdir.exists():
outdir.mkdir()
with UnicodeWriter(outdir.joinpath('languages-and-dialects-geo.csv')) as writer:
writer.writerows(rows)
0
Example 36
Project: glottolog3 Source File: util.py
def update_reflang(args):
stats = Counter()
brugmann_noderefs = jsonload(args.data_dir.joinpath('languoid_refs.json'))
languoid_map = {}
for l in DBSession.query(Languoid).options(joinedload_all(
Language.languageidentifier, LanguageIdentifier.identifier
)):
if l.hid:
languoid_map[l.hid] = l.pk
elif l.iso_code:
languoid_map[l.iso_code] = l.pk
languoid_map[l.id] = l.pk
lgcodes = {}
for rec in get_bib(args):
lgcode = ''
for f in 'lgcode lcode lgcde lgcoe lgcosw'.split():
if rec.get(f):
lgcode = rec[f]
break
if len(lgcode) == 3 or lgcode.startswith('NOCODE_'):
lgcode = '[' + lgcode + ']'
lgcodes[rec.get('glottolog_ref_id', None)] = lgcode
for ref in page_query(
DBSession.query(Ref).order_by(desc(Source.pk)),
n=10000,
commit=True,
verbose=True):
# disregard iso change requests:
if ref.description and ref.description.startswith('Change Request Number '):
stats.update(['ignored'])
continue
if ref.id not in lgcodes:
# remove all language relations for refs no longer in bib!
update_relationship(ref.languages, [])
stats.update(['obsolete'])
continue
language_note = lgcodes[ref.id]
trigger = ca_trigger(language_note)
if trigger:
ref.ca_language_trigger, ref.language_note = trigger
else:
ref.language_note = language_note
remove = brugmann_noderefs['delete'].get(str(ref.pk), [])
# keep relations to non-language languoids:
# FIXME: adapt this for bib-entries now referring to glottocodes of
# families/dialects (e.g. add a sticky-bit to languagesource)
langs = [
l for l in ref.languages if
(l.level != LanguoidLevel.language or not l.active) and l.pk not in remove]
langs_pk = [l.pk for l in langs]
# add relations from filemaker data:
for lpk in brugmann_noderefs['create'].get(str(ref.pk), []):
if lpk not in langs_pk:
l = Languoid.get(lpk, default=None)
if l:
langs.append(l)
langs_pk.append(l.pk)
else:
args.log.warn('brugmann relation for non-existing languoid %s' % lpk)
for code in set(get_codes(ref)):
if code not in languoid_map:
stats.update([code])
continue
lpk = languoid_map[code]
if lpk in remove:
print(ref.name, ref.id, '--', l.name, l.id)
print('relation removed according to brugmann data')
else:
if lpk not in langs_pk:
langs.append(DBSession.query(Languoid).get(lpk))
langs_pk.append(lpk)
a, r = update_relationship(ref.languages, langs)
if a or r:
stats.update(['changed'])
args.log.info('%s' % stats)
0
Example 37
Project: DIRAC Source File: RequestDB.py
def getRequest( self, reqID = 0, assigned = True ):
""" read request for execution
:param reqID: request's ID (default 0) If 0, take a pseudo random one
"""
# expire_on_commit is set to False so that we can still use the object after we close the session
session = self.DBSession( expire_on_commit = False )
log = self.log.getSubLogger( 'getRequest' if assigned else 'peekRequest' )
requestID = None
try:
if reqID:
requestID = reqID
log.verbose( "selecting request '%s'%s" % ( reqID, ' (Assigned)' if assigned else '' ) )
status = None
try:
status = session.query( Request._Status )\
.filter( Request.RequestID == reqID )\
.one()
except NoResultFound, e:
return S_ERROR( "getRequest: request '%s' not exists" % reqID )
if status and status == "Assigned" and assigned:
return S_ERROR( "getRequest: status of request '%s' is 'Assigned', request cannot be selected" % reqID )
else:
now = datetime.datetime.utcnow().replace( microsecond = 0 )
reqIDs = set()
try:
reqAscIDs = session.query( Request.RequestID )\
.filter( Request._Status == 'Waiting' )\
.filter( Request._NotBefore < now )\
.order_by( Request._LastUpdate )\
.limit( 100 )\
.all()
reqIDs = set( [reqID[0] for reqID in reqAscIDs] )
reqDescIDs = session.query( Request.RequestID )\
.filter( Request._Status == 'Waiting' )\
.filter( Request._NotBefore < now )\
.order_by( Request._LastUpdate.desc() )\
.limit( 50 )\
.all()
reqIDs |= set( [reqID[0] for reqID in reqDescIDs] )
# No Waiting requests
except NoResultFound, e:
return S_OK()
if not reqIDs:
return S_OK()
reqIDs = list( reqIDs )
random.shuffle( reqIDs )
requestID = reqIDs[0]
# If we are here, the request MUST exist, so no try catch
# the joinedload_all is to force the non-lazy loading of all the attributes, especially _parent
request = session.query( Request )\
.options( joinedload_all( '__operations__.__files__' ) )\
.filter( Request.RequestID == requestID )\
.one()
if not reqID:
log.verbose( "selected request %s('%s')%s" % ( request.RequestID, request.RequestName, ' (Assigned)' if assigned else '' ) )
if assigned:
session.execute( update( Request )\
.where( Request.RequestID == requestID )\
.values( {Request._Status : 'Assigned',
Request._LastUpdate : datetime.datetime.utcnow()\
.strftime( Request._datetimeFormat )} )
)
session.commit()
session.expunge_all()
return S_OK( request )
except Exception as e:
session.rollback()
log.exception( "getRequest: unexpected exception", lException = e )
return S_ERROR( "getRequest: unexpected exception : %s" % e )
finally:
session.close()
0
Example 38
Project: DIRAC Source File: RequestDB.py
def getBulkRequests( self, numberOfRequest = 10, assigned = True ):
""" read as many requests as requested for execution
:param int numberOfRequest: Number of Request we want (default 10)
:param bool assigned: if True, the status of the selected requests are set to assign
:returns: a dictionary of Request objects indexed on the RequestID
"""
# expire_on_commit is set to False so that we can still use the object after we close the session
session = self.DBSession( expire_on_commit = False )
log = self.log.getSubLogger( 'getBulkRequest' if assigned else 'peekBulkRequest' )
requestDict = {}
try:
# If we are here, the request MUST exist, so no try catch
# the joinedload_all is to force the non-lazy loading of all the attributes, especially _parent
try:
now = datetime.datetime.utcnow().replace( microsecond = 0 )
requestIDs = session.query( Request.RequestID )\
.with_for_update()\
.filter( Request._Status == 'Waiting' )\
.filter( Request._NotBefore < now )\
.order_by( Request._LastUpdate )\
.limit( numberOfRequest )\
.all()
requestIDs = [ridTuple[0] for ridTuple in requestIDs]
log.debug( "Got request ids %s" % requestIDs )
requests = session.query( Request )\
.options( joinedload_all( '__operations__.__files__' ) )\
.filter( Request.RequestID.in_( requestIDs ) )\
.all()
log.debug( "Got %s Request objects " % len( requests ) )
requestDict = dict((req.RequestID, req) for req in requests)
# No Waiting requests
except NoResultFound, e:
pass
if assigned and requestDict:
session.execute( update( Request )\
.where( Request.RequestID.in_( requestDict.keys() ) )\
.values( {Request._Status : 'Assigned',
Request._LastUpdate : datetime.datetime.utcnow()\
.strftime( Request._datetimeFormat )} )
)
session.commit()
session.expunge_all()
except Exception as e:
session.rollback()
log.exception( "unexpected exception", lException = e )
return S_ERROR( "getBulkRequest: unexpected exception : %s" % e )
finally:
session.close()
return S_OK( requestDict )
0
Example 39
def readRequestsForJobs( self, jobIDs = None ):
""" read request for jobs
:param list jobIDs: list of JobIDs
:return: S_OK( "Successful" : { jobID1 : Request, jobID2: Request, ... }
"Failed" : { jobID3: "error message", ... } )
"""
self.log.debug( "readRequestForJobs: got %s jobIDs to check" % str( jobIDs ) )
if not jobIDs:
return S_ERROR( "Must provide jobID list as argument." )
if isinstance( jobIDs, ( long, int ) ):
jobIDs = [ jobIDs ]
jobIDs = set( jobIDs )
reqDict = { "Successful": {}, "Failed": {} }
# expire_on_commit is set to False so that we can still use the object after we close the session
session = self.DBSession( expire_on_commit = False )
try:
ret = session.query( Request.JobID, Request )\
.options( joinedload_all( '__operations__.__files__' ) )\
.filter( Request.JobID.in_( jobIDs ) ).all()
reqDict['Successful'] = dict( ( jobId, reqObj ) for jobId, reqObj in ret )
reqDict['Failed'] = dict( ( jobid, 'Request not found' ) for jobid in jobIDs - set( reqDict['Successful'] ) )
session.expunge_all()
except Exception as e:
self.log.exception( "readRequestsForJobs: unexpected exception", lException = e )
return S_ERROR( "readRequestsForJobs: unexpected exception : %s" % e )
finally:
session.close()
return S_OK( reqDict )
0
Example 40
def check_certificate(self, request):
"""Check a client certificate serial and add authentication if valid."""
self.state.pop('cert_serial', None)
serial = get_certificate_serial(request)
if not serial:
# No cert. Our obligation to wipe cert state is fulfilled above.
return
# Figure out what certificate and user this serial belongs to
# TODO: Optimize eagerloading
serial = serial.lower()
q = model.session.query(model.Certificate) \
.options(joinedload_all('user.roles')) \
.filter_by(serial=serial)
try:
cert = q.one()
except NoResultFound:
raise CertNotFoundError
if cert.user.cert_auth == u'disabled':
raise CertAuthDisabledError
if cert.expired:
raise CertExpiredError
if cert.revoked:
raise CertRevokedError
if self.user and self.user.id != cert.user_id:
raise AuthConflictError
# At this point, we're confident that the supplied cert is valid
self.state['cert_serial'] = serial
self.trust.append('cert')
if not self.user:
self.user = cert.user
0
Example 41
def check_openid(self, config):
"""Check OpenID state and add authentication if valid, else clear."""
url = self.state.pop('openid_url', None)
timestamp = self.state.pop('openid_timestamp', None)
if not url or timestamp is None:
# No (or corrupted) OpenID login. By popping, our obligation to
# clear relevent state is already fulfilled, so just return
return
# TODO: Optimize eagerloading
q = model.session.query(model.IdentityURL) \
.options(joinedload_all('user.roles')) \
.filter_by(url=url)
try:
openid = q.one()
except NoResultFound:
raise OpenIDNotFoundError
if openid.user.cert_auth == 'required':
raise OpenIDAuthDisabledError
if self.user and self.user.id != openid.user_id:
raise AuthConflictError
# XXX Check timestamp sanity?
# At this point, we're confident that the stored OpenID login is valid
self.state['openid_url'] = url
self.state['openid_timestamp'] = timestamp
self.trust.append('openid')
# Evaluate OpenID freshness
confidence_expiry_secs = int(config.get(
'auth.openid.expiry_seconds',
DEFAULT_CONFIDENCE_EXPIRY))
age = datetime.utcnow() - datetime.utcfromtimestamp(timestamp)
if age <= timedelta(seconds=confidence_expiry_secs):
self.trust.append('openid_recent')
if not self.user:
self.user = openid.user
0
Example 42
Project: floof Source File: authn.py
def check_persona(self, config):
"""Check Persona state and add authentication if valid, else
clear."""
# XXX this is very similar to check_openid() above
addr = self.state.pop('persona_addr', None)
timestamp = self.state.pop('persona_timestamp', None)
if not addr or not timestamp:
# No (or corrupted) Persona login. By popping, our obligation to
# clear relevent state is already fulfilled, so just return
return
# TODO: Optimize eagerloading
q = model.session.query(model.IdentityEmail) \
.options(joinedload_all('user.roles')) \
.filter_by(email=addr)
try:
persona = q.one()
except NoResultFound:
raise PersonaNotFoundError
if persona.user.cert_auth == 'required':
raise PersonaAuthDisabledError
if self.user and self.user.id != persona.user.id:
raise AuthConflictError
# At this point, we're confident that the stored Persona login is valid
self.state['persona_addr'] = addr
self.state['persona_timestamp'] = timestamp
self.trust.append('persona')
# Evaluate Persona freshness
confidence_expiry_secs = int(config.get(
'auth.persona.expiry_seconds',
DEFAULT_CONFIDENCE_EXPIRY))
age = datetime.utcnow() - datetime.utcfromtimestamp(timestamp)
if age <= timedelta(seconds=confidence_expiry_secs):
self.trust.append('persona_recent')
if not self.user:
self.user = persona.user
0
Example 43
Project: Flexget Source File: perf_tests.py
def imdb_query(session):
import time
from flexget.plugins.metainfo.imdb_lookup import Movie
from flexget.plugins.cli.performance import log_query_count
from sqlalchemy.sql.expression import select
from progressbar import ProgressBar, Percentage, Bar, ETA
from sqlalchemy.orm import joinedload_all
imdb_urls = []
log.info('Getting imdb_urls ...')
# query so that we avoid loading whole object (maybe cached?)
for _, url in session.execute(select([Movie.id, Movie.url])):
imdb_urls.append(url)
log.info('Got %i urls from database' % len(imdb_urls))
if not imdb_urls:
log.info('so .. aborting')
return
# commence testing
widgets = ['Benchmarking - ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')]
bar = ProgressBar(widgets=widgets, maxval=len(imdb_urls)).start()
log_query_count('test')
start_time = time.time()
for index, url in enumerate(imdb_urls):
bar.update(index)
# movie = session.query(Movie).filter(Movie.url == url).first()
# movie = session.query(Movie).options(subqueryload(Movie.genres)).filter(Movie.url == url).one()
movie = session.query(Movie). \
options(joinedload_all(Movie.genres, Movie.languages,
Movie.actors, Movie.directors)). \
filter(Movie.url == url).first()
# access it's members so they're loaded
[x.name for x in movie.genres]
[x.name for x in movie.directors]
[x.name for x in movie.actors]
[x.name for x in movie.languages]
log_query_count('test')
took = time.time() - start_time
log.debug('Took %.2f seconds to query %i movies' % (took, len(imdb_urls)))
0
Example 44
def model_query(context, *args):
with read_session() as session:
query = session.query(*args).options(joinedload_all('*'))
return query