sqlalchemy.orm.joinedload_all

Here are the examples of the python api sqlalchemy.orm.joinedload_all taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.

44 Examples 7

Example 1

Project: quicktill Source File: views.py
@tillweb_view
def transaction(request,info,session,transid):
    try:
        t=session.query(Transaction).\
            filter_by(id=int(transid)).\
            options(subqueryload_all('payments')).\
            options(joinedload_all('lines.stockref.stockitem.stocktype')).\
            options(joinedload('lines.user')).\
            options(undefer('total')).\
            one()
    except NoResultFound:
        raise Http404
    return ('transaction.html',{'transaction':t,})

Example 2

Project: quicktill Source File: views.py
@tillweb_view
def transline(request,info,session,translineid):
    try:
        tl=session.query(Transline).\
            filter_by(id=int(translineid)).\
            options(joinedload_all('stockref.stockitem.stocktype')).\
            options(joinedload('user')).\
            one()
    except NoResultFound:
        raise Http404
    return ('transline.html',{'tl':tl,})

Example 3

Project: quicktill Source File: views.py
Function: delivery
@tillweb_view
def delivery(request,info,session,deliveryid):
    try:
        d=session.query(Delivery).\
            filter_by(id=int(deliveryid)).\
            options(joinedload_all('items.stocktype.unit')).\
            options(joinedload_all('items.stockline')).\
            options(undefer_group('qtys')).\
            one()
    except NoResultFound:
        raise Http404
    return ('delivery.html',{'delivery':d,})

Example 4

Project: quicktill Source File: views.py
@tillweb_view
def stocksearch(request,info,session):
    form=StockForm(request.GET)
    result=[]
    q=session.query(StockItem).join(StockType).order_by(StockItem.id).\
        options(joinedload_all('stocktype.unit')).\
        options(joinedload('stockline')).\
        options(undefer_group('qtys'))
    if form.is_valid():
        if form.is_filled_in():
            q=form.filter(q)
            if not form.cleaned_data['include_finished']:
                q=q.filter(StockItem.finished==None)
            result=q.all()
    return ('stocksearch.html',{'form':form,'stocklist':result})

Example 5

Project: quicktill Source File: views.py
Function: stock
@tillweb_view
def stock(request,info,session,stockid):
    try:
        s=session.query(StockItem).\
            filter_by(id=int(stockid)).\
            options(joinedload_all('stocktype.department')).\
            options(joinedload_all('stocktype.stockline_log.stockline')).\
            options(joinedload_all('delivery.supplier')).\
            options(joinedload_all('stockunit.unit')).\
            options(joinedload_all('annotations.type')).\
            options(subqueryload_all('out.transline.transaction')).\
            options(undefer_group('qtys')).\
            one()
    except NoResultFound:
        raise Http404
    return ('stock.html',{'stock':s,})

Example 6

Project: quicktill Source File: views.py
@tillweb_view
def stockline(request,info,session,stocklineid):
    try:
        s=session.query(StockLine).\
            filter_by(id=int(stocklineid)).\
            options(joinedload_all('stockonsale.stocktype.unit')).\
            options(undefer_group('qtys')).\
            one()
    except NoResultFound:
        raise Http404
    return ('stockline.html',{'stockline':s,})

Example 7

Project: quicktill Source File: views.py
@tillweb_view
def department(request,info,session,departmentid):
    d=session.query(Department).get(int(departmentid))
    if d is None: raise Http404
    include_finished=request.GET.get("show_finished","off")=="on"
    items=session.query(StockItem).\
        join(StockType).\
        filter(StockType.department==d).\
        order_by(desc(StockItem.id)).\
        options(joinedload_all('stocktype.unit')).\
        options(undefer_group('qtys')).\
        options(joinedload('stockline')).\
        options(joinedload('finishcode'))
    if not include_finished:
        items=items.filter(StockItem.finished==None)
    items=items.all()
    return ('department.html',{'department':d,'items':items,
                               'include_finished':include_finished})

Example 8

Project: baruwa2 Source File: base.py
    def _get_domain(self, domainid):
        "utility to return domain"
        try:
            cachekey = 'domain-%s' % domainid
            qry = Session.query(Domain).filter(Domain.id == domainid)\
                    .options(joinedload_all(Domain.servers),
                            joinedload_all(Domain.aliases),
                            joinedload_all(Domain.authservers))\
                    .options(FromCache('sql_cache_med', cachekey))
            if self.invalidate:
                qry.invalidate()
            domain = qry.one()
        except NoResultFound:
            domain = None
        return domain

Example 9

Project: reggata Source File: commands.py
    def _execute(self, uow):
        self._session = uow.session
        item = self._session.query(db.Item)\
            .options(joinedload_all('data_ref'))\
            .options(joinedload_all('item_tags.tag'))\
            .options(joinedload_all('item_fields.field'))\
            .get(self.__itemId)

        if item is None:
            raise err.NotFoundError()

        self._session.expunge(item)
        return item

Example 10

Project: sqlalchemy Source File: test_of_type.py
    def test_twolevel_joinedload_wsubclass(self):
        ParentThing, DataContainer, Job, SubJob = \
            self.classes.ParentThing,\
            self.classes.DataContainer,\
            self.classes.Job,\
            self.classes.SubJob
        s = Session(testing.db)
        q = s.query(ParentThing).\
                        options(
                            joinedload_all(
                                ParentThing.container,
                                DataContainer.jobs.of_type(SubJob)
                        ))
        def go():
            eq_(
                q.all(),
                self._fixture()
            )
        self.assert_sql_count(testing.db, go, 5)

Example 11

Project: sqlalchemy Source File: test_options.py
    def test_option_against_nonexistent_twolevel_all(self):
        Item = self.classes.Item
        self._assert_eager_with_entity_exception(
            [Item],
            (joinedload_all("keywords.foo"), ),
            r"Can't find property named 'foo' on the mapped entity "
            r"Mapper\|Keyword\|keywords in this Query."
        )

Example 12

Project: sqlalchemy Source File: test_options.py
    @testing.fails_if(lambda: True,
        "PropertyOption doesn't yet check for relation/column on end result")
    def test_option_against_non_relation_basestring(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword, Item],
            (joinedload_all("keywords"), ),
            r"Attribute 'keywords' of entity 'Mapper\|Keyword\|keywords' "
            "does not refer to a mapped entity"
        )

Example 13

Project: sqlalchemy Source File: test_options.py
    @testing.fails_if(lambda: True,
            "PropertyOption doesn't yet check for relation/column on end result")
    def test_option_against_multi_non_relation_basestring(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword, Item],
            (joinedload_all("keywords"), ),
            r"Attribute 'keywords' of entity 'Mapper\|Keyword\|keywords' "
            "does not refer to a mapped entity"
        )

Example 14

Project: sqlalchemy Source File: test_options.py
    def test_option_against_wrong_entity_type_basestring(self):
        Item = self.classes.Item
        self._assert_eager_with_entity_exception(
            [Item],
            (joinedload_all("id", "keywords"), ),
            r"Attribute 'id' of entity 'Mapper\|Item\|items' does not "
            r"refer to a mapped entity"
        )

Example 15

Project: sqlalchemy Source File: test_options.py
    def test_option_against_multi_non_relation_twolevel_basestring(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword, Item],
            (joinedload_all("id", "keywords"), ),
            r"Attribute 'id' of entity 'Mapper\|Keyword\|keywords' "
            "does not refer to a mapped entity"
        )

Example 16

Project: sqlalchemy Source File: test_options.py
    def test_option_against_multi_nonexistent_basestring(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword, Item],
            (joinedload_all("description"), ),
            r"Can't find property named 'description' on the mapped "
            r"entity Mapper\|Keyword\|keywords in this Query."
        )

Example 17

Project: sqlalchemy Source File: test_options.py
    def test_option_against_multi_no_entities_basestring(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword.id, Item.id],
            (joinedload_all("keywords"), ),
            r"Query has only expression-based entities - can't find property "
            "named 'keywords'."
        )

Example 18

Project: sqlalchemy Source File: test_options.py
    def test_option_against_wrong_multi_entity_type_attr_one(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword, Item],
            (joinedload_all(Keyword.id, Item.keywords), ),
            r"Attribute 'id' of entity 'Mapper\|Keyword\|keywords' "
            "does not refer to a mapped entity"
        )

Example 19

Project: sqlalchemy Source File: test_options.py
    def test_option_against_wrong_multi_entity_type_attr_two(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword, Item],
            (joinedload_all(Keyword.keywords, Item.keywords), ),
            r"Attribute 'keywords' of entity 'Mapper\|Keyword\|keywords' "
            "does not refer to a mapped entity"
        )

Example 20

Project: sqlalchemy Source File: test_options.py
    def test_option_against_wrong_multi_entity_type_attr_three(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Keyword.id, Item.id],
            (joinedload_all(Keyword.keywords, Item.keywords), ),
            r"Query has only expression-based entities - "
            "can't find property named 'keywords'."
        )

Example 21

Project: sqlalchemy Source File: test_options.py
    def test_wrong_type_in_option(self):
        Item = self.classes.Item
        Keyword = self.classes.Keyword
        self._assert_eager_with_entity_exception(
            [Item],
            (joinedload_all(Keyword), ),
            r"mapper option expects string key or list of attributes"
        )

Example 22

Project: sqlalchemy Source File: test_options.py
    def test_non_contiguous_all_option(self):
        User = self.classes.User
        self._assert_eager_with_entity_exception(
            [User],
            (joinedload_all(User.addresses, User.orders), ),
            r"Attribute 'User.orders' does not link "
            "from element 'Mapper|Address|addresses'"
        )

Example 23

Project: sqlalchemy Source File: test_options.py
    def test_non_contiguous_all_option_of_type(self):
        User = self.classes.User
        Order = self.classes.Order
        self._assert_eager_with_entity_exception(
            [User],
            (joinedload_all(User.addresses, User.orders.of_type(Order)), ),
            r"Attribute 'User.orders' does not link "
            "from element 'Mapper|Address|addresses'"
        )

Example 24

Project: glottolog3 Source File: update_alternative_names.py
Function: main
def main(args):  # pragma: no cover
    global MAX_IDENTIFIER_PK
    stats = Counter()

    with transaction.manager:
        MAX_IDENTIFIER_PK = DBSession.query(
            Identifier.pk).order_by(desc(Identifier.pk)).first()[0]

        gl_names = glottolog_names()

        for l in DBSession.query(Languoid).options(joinedload_all(
            Language.languageidentifier, LanguageIdentifier.identifier
        )):
            stats.update(create_name(gl_names, l))
    args.log.info('%s' % stats)

Example 25

Project: glottolog3 Source File: __init__.py
    def refined_query(self, query, model, req):
        if model == Language:
            query = query.options(
                joinedload(models.Languoid.family),
                joinedload(models.Languoid.children),
                joinedload_all(
                    Language.valuesets, ValueSet.references, ValueSetReference.source)
            )
        return query

Example 26

Project: ggrc-core Source File: attribute_query.py
  def process_eager_loading(self, value):
    paths = value.split(',')
    options = []
    for path in paths:
      segments = path.split('.')
      real_segments = []
      current_model = self.model
      for segment in segments:
        real_segment, current_model = self.resolve_path_segment(
          segment, current_model)
        if current_model is not None:
          real_segments.append(real_segment)
      realized_path = '.'.join(real_segments)
      options.append(joinedload_all(realized_path))
    return options

Example 27

Project: quicktill Source File: managestock.py
@user.permission_required('stock-history', 'List finished stock')
def stockhistory(dept=None):
    log.info("Stock history")
    sq = td.s.query(StockItem)\
             .join(StockItem.stocktype)\
             .filter(StockItem.finished != None)\
             .options(undefer(StockItem.remaining))\
             .options(joinedload_all('stocktype.unit'))\
             .order_by(StockItem.id.desc())
    if dept:
        sq = sq.filter(StockType.dept_id == dept)
    sinfo = sq.all()
    f = ui.tableformatter(' r l l ')
    sl = [(f(x.id, x.stocktype.format(), x.remaining_units),
           stock.stockinfo_popup, (x.id,)) for x in sinfo]
    title = "Stock History" if dept is None \
            else "Stock History department {}".format(dept)
    ui.menu(sl, title=title, blurb="Select a stock item and press "
            "Cash/Enter for more information.  The number of units remaining "
            "when the stock was finished is shown.", dismiss_on_select=False)

Example 28

Project: quicktill Source File: views.py
@tillweb_view
def pubroot(request,info,session):
    date=datetime.date.today()
    # If it's the early hours of the morning, it's more useful for us
    # to consider it still to be yesterday.
    if datetime.datetime.now().hour<4: date=date-datetime.timedelta(1)
    thisweek_start=date-datetime.timedelta(date.weekday())
    thisweek_end=thisweek_start+datetime.timedelta(6)
    lastweek_start=thisweek_start-datetime.timedelta(7)
    lastweek_end=thisweek_end-datetime.timedelta(7)
    weekbefore_start=lastweek_start-datetime.timedelta(7)
    weekbefore_end=lastweek_end-datetime.timedelta(7)

    weeks=[("Current week",thisweek_start,thisweek_end,
            business_totals(session,thisweek_start,thisweek_end)),
           ("Last week",lastweek_start,lastweek_end,
            business_totals(session,lastweek_start,lastweek_end)),
           ("The week before last",weekbefore_start,weekbefore_end,
            business_totals(session,weekbefore_start,weekbefore_end))]

    currentsession=Session.current(session)
    barsummary=session.query(StockLine).\
        filter(StockLine.location=="Bar").\
        order_by(StockLine.dept_id,StockLine.name).\
        options(joinedload_all('stockonsale.stocktype.unit')).\
        options(undefer_group('qtys')).\
        all()
    stillage=session.query(StockAnnotation).\
        join(StockItem).\
        outerjoin(StockLine).\
        filter(tuple_(StockAnnotation.text,StockAnnotation.time).in_(
            select([StockAnnotation.text,func.max(StockAnnotation.time)],
                   StockAnnotation.atype=='location').\
                group_by(StockAnnotation.text))).\
        filter(StockItem.finished==None).\
        order_by(StockLine.name!=null(),StockAnnotation.time).\
        options(joinedload_all('stockitem.stocktype.unit')).\
        options(joinedload_all('stockitem.stockline')).\
        options(undefer_group('qtys')).\
        all()
    return ('index.html',
            {'currentsession':currentsession,
             'barsummary':barsummary,
             'stillage':stillage,
             'weeks':weeks,
             })

Example 29

Project: quicktill Source File: views.py
@tillweb_view
def sessiondept(request,info,session,sessionid,dept):
    try:
        s=session.query(Session).filter_by(id=int(sessionid)).one()
    except NoResultFound:
        raise Http404
    try:
        dept=session.query(Department).filter_by(id=int(dept)).one()
    except NoResultFound:
        raise Http404
    nextsession=session.query(Session).\
        filter(Session.id>s.id).\
        order_by(Session.id).\
        first()
    nextlink=info['base']+nextsession.tillweb_url+"dept{}/".format(dept.id) \
        if nextsession else None
    prevsession=session.query(Session).\
        filter(Session.id<s.id).\
        order_by(desc(Session.id)).\
        first()
    prevlink=info['base']+prevsession.tillweb_url+"dept{}/".format(dept.id) \
        if prevsession else None
    translines=session.query(Transline).\
        join(Transaction).\
        options(joinedload('transaction')).\
        options(joinedload('user')).\
        options(joinedload_all('stockref.stockitem.stocktype.unit')).\
        filter(Transaction.sessionid==s.id).\
        filter(Transline.dept_id==dept.id).\
        order_by(Transline.id).\
        all()
    return ('sessiondept.html',{'session':s,'department':dept,
                                'translines':translines,
                                'nextlink':nextlink,'prevlink':prevlink})

Example 30

Project: quicktill Source File: views.py
Function: user
@tillweb_view
def user(request,info,session,userid):
    try:
        u=session.query(User).\
            options(joinedload('permissions')).\
            options(joinedload('tokens')).\
            get(int(userid))
    except NoResultFound:
        raise Http404
    sales=session.query(Transline).filter(Transline.user==u).\
        options(joinedload('transaction')).\
        options(joinedload_all('stockref.stockitem.stocktype.unit')).\
        order_by(desc(Transline.time))[:50]
    payments=session.query(Payment).filter(Payment.user==u).\
        options(joinedload('transaction')).\
        options(joinedload('paytype')).\
        order_by(desc(Payment.time))[:50]
    annotations=session.query(StockAnnotation).\
        options(joinedload_all('stockitem.stocktype')).\
        options(joinedload('type')).\
        filter(StockAnnotation.user==u).\
        order_by(desc(StockAnnotation.time))[:50]
    return ('user.html',{'tuser':u,'sales':sales,'payments':payments,
                         'annotations':annotations})

Example 31

Project: reggata Source File: commands.py
    def __getFileInfo(self, path):
        try:
            data_ref = self._session.query(db.DataRef)\
                .filter(db.DataRef.url_raw==hlp.to_db_format(path))\
                .options(joinedload_all("items"))\
                .options(joinedload_all("items.item_tags.tag"))\
                .options(joinedload_all("items.item_fields.field"))\
                .one()
            self._session.expunge(data_ref)
            finfo = FileInfo(data_ref.url, objType=FileInfo.FILE, status=FileInfo.STORED)

            for item in data_ref.items:
                finfo.itemIds.append(item.id)
                for item_tag in item.item_tags:
                    finfo.tags.append(item_tag.tag.name)
                for item_field in item.item_fields:
                    finfo.fields.append((item_field.field.name, item_field.field_value))
            return finfo

        except NoResultFound:
            return FileInfo(self.__relPath, status=FileInfo.UNTRACKED)

Example 32

Project: sqlalchemy Source File: test_assorted_eager.py
    def test_joinedload_on_path(self):
        Entry, Account, Transaction = (self.classes.Entry,
                                self.classes.Account,
                                self.classes.Transaction)

        session = create_session()

        tx1 = Transaction(name='tx1')
        tx2 = Transaction(name='tx2')

        acc1 = Account(name='acc1')
        ent11 = Entry(name='ent11', account=acc1, transaction=tx1)
        ent12 = Entry(name='ent12', account=acc1, transaction=tx2)

        acc2 = Account(name='acc2')
        ent21 = Entry(name='ent21', account=acc2, transaction=tx1)
        ent22 = Entry(name='ent22', account=acc2, transaction=tx2)

        session.add(acc1)
        session.flush()
        session.expunge_all()

        def go():
            # load just the first Account.  eager loading will actually load
            # all objects saved thus far, but will not eagerly load the
            # "accounts" off the immediate "entries"; only the "accounts" off
            # the entries->transaction->entries
            acc = (session.query(Account).
                   options(sa.orm.joinedload_all('entries.transaction.entries.account')).
                   order_by(Account.account_id)).first()

            # no sql occurs
            eq_(acc.name, 'acc1')
            eq_(acc.entries[0].transaction.entries[0].account.name, 'acc1')
            eq_(acc.entries[0].transaction.entries[1].account.name, 'acc2')

            # lazyload triggers but no sql occurs because many-to-one uses
            # cached query.get()
            for e in acc.entries:
                assert e.account is acc

        self.assert_sql_count(testing.db, go, 1)

Example 33

Project: glottolog3 Source File: adapters.py
Function: query
    def query(self, req):
        return req.db.query(Language).options(joinedload_all(
            Language.languageidentifier, LanguageIdentifier.identifier))\
            .order_by(Language.pk)

Example 34

Project: glottolog3 Source File: langdocstatus.py
def main(args):  # pragma: no cover
    # we merge information about extinct languages from unesco and Harald.
    extinct = dict(list(dsv.reader(args.data_file('extinct.tab'))))
    with transaction.manager:
        query = language_query().options(
            joinedload_all(Language.valuesets, ValueSet.values))
        # loop over active, established languages with geo-coords
        for l in page_query(query, n=100, verbose=True):
            # let's collect the relevant sources in a way that allows computation of med.
            # Note: we limit refs to the ones without computerized assignments.
            sources = DBSession.query(Ref).join(LanguageSource)\
                .filter(LanguageSource.language_pk == l.pk) \
                .filter(Ref.ca_doctype_trigger == None)\
                .filter(Ref.ca_language_trigger == None)\
                .options(joinedload(Ref.doctypes))
            sources = sorted(map(Source, sources))

            # keep the overall med
            # note: this source may not be included in the potential meds computed below,
            # e.g. because it may not have a year.
            med = sources[0].__json__() if sources else None

            # now we have to compute meds respecting a cut-off year.
            # to do so, we collect eligible sources per year and then
            # take the med of this collection.
            potential_meds = []

            # we only have to loop over publication years within all sources, because
            # only in these years something better might have come along.
            for year in set(s.year for s in sources if s.year):
                # let's see if something better was published!
                eligible = [s for s in sources if s.year and s.year <= year]
                if eligible:
                    potential_meds.append(sorted(eligible)[0])

            # we store the precomputed sources information as jsondata:
            l.update_jsondata(
                endangerment='Extinct' if l.hid in extinct else l.endangerment,
                med=med,
                sources=[s.__json__() for s in
                         sorted(set(potential_meds), key=lambda s: -s.year)])

Example 35

Project: glottolog3 Source File: languages_and_dialects_geo_download.py
def main(args):  # pragma: no cover
    rows = [
        ['glottocode', 'name', 'isocodes', 'level', 'macroarea', 'latitude', 'longitude']]
    version_pattern = re.compile('glottolog\-(?P<version>[0-9]+\.[0-9]+)')

    with transaction.manager:
        # loop over languages and dialects
        for l in DBSession.query(Languoid)\
                .filter(Language.active)\
                .filter(or_(
                    Languoid.level == LanguoidLevel.dialect,
                    and_(
                        Languoid.level == LanguoidLevel.language,
                        Languoid.status == LanguoidStatus.established)))\
                .options(
                    joinedload(Languoid.macroareas),
                    joinedload_all(
                        Language.languageidentifier, LanguageIdentifier.identifier))\
                .order_by(Language.name):
            rows.append([
                l.id,
                l.name,
                ' '.join(i.name for i in l.get_identifier_objs(IdentifierType.iso)),
                l.level,
                l.macroareas[0].name if l.macroareas else '',
                l.latitude if l.latitude is not None else '',
                l.longitude if l.longitude is not None else ''])

    outdir = args.module_dir.joinpath('static', 'download')
    match = version_pattern.search(args.config_uri)
    if match:
        outdir = outdir.joinpath(match.group('version'))
        if not outdir.exists():
            outdir.mkdir()
    with UnicodeWriter(outdir.joinpath('languages-and-dialects-geo.csv')) as writer:
        writer.writerows(rows)

Example 36

Project: glottolog3 Source File: util.py
def update_reflang(args):
    stats = Counter()
    brugmann_noderefs = jsonload(args.data_dir.joinpath('languoid_refs.json'))

    languoid_map = {}
    for l in DBSession.query(Languoid).options(joinedload_all(
        Language.languageidentifier, LanguageIdentifier.identifier
    )):
        if l.hid:
            languoid_map[l.hid] = l.pk
        elif l.iso_code:
            languoid_map[l.iso_code] = l.pk
        languoid_map[l.id] = l.pk

    lgcodes = {}
    for rec in get_bib(args):
        lgcode = ''
        for f in 'lgcode lcode lgcde lgcoe lgcosw'.split():
            if rec.get(f):
                lgcode = rec[f]
                break
        if len(lgcode) == 3 or lgcode.startswith('NOCODE_'):
            lgcode = '[' + lgcode + ']'
        lgcodes[rec.get('glottolog_ref_id', None)] = lgcode

    for ref in page_query(
            DBSession.query(Ref).order_by(desc(Source.pk)),
            n=10000,
            commit=True,
            verbose=True):
        # disregard iso change requests:
        if ref.description and ref.description.startswith('Change Request Number '):
            stats.update(['ignored'])
            continue

        if ref.id not in lgcodes:
            # remove all language relations for refs no longer in bib!
            update_relationship(ref.languages, [])
            stats.update(['obsolete'])
            continue

        language_note = lgcodes[ref.id]
        trigger = ca_trigger(language_note)
        if trigger:
            ref.ca_language_trigger, ref.language_note = trigger
        else:
            ref.language_note = language_note

        remove = brugmann_noderefs['delete'].get(str(ref.pk), [])

        # keep relations to non-language languoids:
        # FIXME: adapt this for bib-entries now referring to glottocodes of
        #        families/dialects (e.g. add a sticky-bit to languagesource)
        langs = [
            l for l in ref.languages if
            (l.level != LanguoidLevel.language or not l.active) and l.pk not in remove]
        langs_pk = [l.pk for l in langs]

        # add relations from filemaker data:
        for lpk in brugmann_noderefs['create'].get(str(ref.pk), []):
            if lpk not in langs_pk:
                l = Languoid.get(lpk, default=None)
                if l:
                    langs.append(l)
                    langs_pk.append(l.pk)
                else:
                    args.log.warn('brugmann relation for non-existing languoid %s' % lpk)

        for code in set(get_codes(ref)):
            if code not in languoid_map:
                stats.update([code])
                continue
            lpk = languoid_map[code]
            if lpk in remove:
                print(ref.name, ref.id, '--', l.name, l.id)
                print('relation removed according to brugmann data')
            else:
                if lpk not in langs_pk:
                    langs.append(DBSession.query(Languoid).get(lpk))
                    langs_pk.append(lpk)

        a, r = update_relationship(ref.languages, langs)
        if a or r:
            stats.update(['changed'])

    args.log.info('%s' % stats)

Example 37

Project: DIRAC Source File: RequestDB.py
  def getRequest( self, reqID = 0, assigned = True ):
    """ read request for execution

    :param reqID: request's ID (default 0) If 0, take a pseudo random one

    """

    # expire_on_commit is set to False so that we can still use the object after we close the session
    session = self.DBSession( expire_on_commit = False )
    log = self.log.getSubLogger( 'getRequest' if assigned else 'peekRequest' )

    requestID = None

    try:

      if reqID:
        requestID = reqID

        log.verbose( "selecting request '%s'%s" % ( reqID, ' (Assigned)' if assigned else '' ) )
        status = None
        try:
          status = session.query( Request._Status )\
                          .filter( Request.RequestID == reqID )\
                          .one()
        except NoResultFound, e:
          return S_ERROR( "getRequest: request '%s' not exists" % reqID )

        if status and status == "Assigned" and assigned:
          return S_ERROR( "getRequest: status of request '%s' is 'Assigned', request cannot be selected" % reqID )

      else:
        now = datetime.datetime.utcnow().replace( microsecond = 0 )
        reqIDs = set()
        try:
          reqAscIDs = session.query( Request.RequestID )\
                             .filter( Request._Status == 'Waiting' )\
                             .filter( Request._NotBefore < now )\
                             .order_by( Request._LastUpdate )\
                             .limit( 100 )\
                             .all()

          reqIDs = set( [reqID[0] for reqID in reqAscIDs] )

          reqDescIDs = session.query( Request.RequestID )\
                              .filter( Request._Status == 'Waiting' )\
                              .filter( Request._NotBefore < now )\
                              .order_by( Request._LastUpdate.desc() )\
                              .limit( 50 )\
                              .all()

          reqIDs |= set( [reqID[0] for reqID in reqDescIDs] )
        # No Waiting requests
        except NoResultFound, e:
          return S_OK()

        if not reqIDs:
          return S_OK()

        reqIDs = list( reqIDs )
        random.shuffle( reqIDs )
        requestID = reqIDs[0]


      # If we are here, the request MUST exist, so no try catch
      # the joinedload_all is to force the non-lazy loading of all the attributes, especially _parent
      request = session.query( Request )\
                       .options( joinedload_all( '__operations__.__files__' ) )\
                       .filter( Request.RequestID == requestID )\
                       .one()

      if not reqID:
        log.verbose( "selected request %s('%s')%s" % ( request.RequestID, request.RequestName, ' (Assigned)' if assigned else '' ) )


      if assigned:
        session.execute( update( Request )\
                         .where( Request.RequestID == requestID )\
                         .values( {Request._Status : 'Assigned',
                                   Request._LastUpdate : datetime.datetime.utcnow()\
                                                        .strftime( Request._datetimeFormat )} )
                       )
        session.commit()

      session.expunge_all()
      return S_OK( request )

    except Exception as e:
      session.rollback()
      log.exception( "getRequest: unexpected exception", lException = e )
      return S_ERROR( "getRequest: unexpected exception : %s" % e )
    finally:
      session.close()

Example 38

Project: DIRAC Source File: RequestDB.py
  def getBulkRequests( self, numberOfRequest = 10, assigned = True ):
    """ read as many requests as requested for execution

    :param int numberOfRequest: Number of Request we want (default 10)
    :param bool assigned: if True, the status of the selected requests are set to assign

    :returns: a dictionary of Request objects indexed on the RequestID

    """

    # expire_on_commit is set to False so that we can still use the object after we close the session
    session = self.DBSession( expire_on_commit = False )
    log = self.log.getSubLogger( 'getBulkRequest' if assigned else 'peekBulkRequest' )

    requestDict = {}

    try:
      # If we are here, the request MUST exist, so no try catch
      # the joinedload_all is to force the non-lazy loading of all the attributes, especially _parent
      try:
        now = datetime.datetime.utcnow().replace( microsecond = 0 )
        requestIDs = session.query( Request.RequestID )\
                          .with_for_update()\
                          .filter( Request._Status == 'Waiting' )\
                          .filter( Request._NotBefore < now )\
                          .order_by( Request._LastUpdate )\
                          .limit( numberOfRequest )\
                          .all()

        requestIDs = [ridTuple[0] for ridTuple in requestIDs]
        log.debug( "Got request ids %s" % requestIDs )

        requests = session.query( Request )\
                          .options( joinedload_all( '__operations__.__files__' ) )\
                          .filter( Request.RequestID.in_( requestIDs ) )\
                          .all()
        log.debug( "Got %s Request objects " % len( requests ) )
        requestDict = dict((req.RequestID, req) for req in requests)
      # No Waiting requests
      except NoResultFound, e:
        pass

      if assigned and requestDict:
        session.execute( update( Request )\
                         .where( Request.RequestID.in_( requestDict.keys() ) )\
                         .values( {Request._Status : 'Assigned',
                                   Request._LastUpdate : datetime.datetime.utcnow()\
                                                        .strftime( Request._datetimeFormat )} )
                       )
      session.commit()

      session.expunge_all()

    except Exception as e:
      session.rollback()
      log.exception( "unexpected exception", lException = e )
      return S_ERROR( "getBulkRequest: unexpected exception : %s" % e )
    finally:
      session.close()

    return S_OK( requestDict )

Example 39

Project: DIRAC Source File: RequestDB.py
Function: readrequestsforjobs
  def readRequestsForJobs( self, jobIDs = None ):
    """ read request for jobs

    :param list jobIDs: list of JobIDs
    :return: S_OK( "Successful" : { jobID1 : Request, jobID2: Request, ... }
                   "Failed" : { jobID3: "error message", ... } )
    """
    self.log.debug( "readRequestForJobs: got %s jobIDs to check" % str( jobIDs ) )
    if not jobIDs:
      return S_ERROR( "Must provide jobID list as argument." )
    if isinstance( jobIDs, ( long, int ) ):
      jobIDs = [ jobIDs ]
    jobIDs = set( jobIDs )

    reqDict = { "Successful": {}, "Failed": {} }

    # expire_on_commit is set to False so that we can still use the object after we close the session
    session = self.DBSession( expire_on_commit = False )

    try:
      ret = session.query( Request.JobID, Request )\
                   .options( joinedload_all( '__operations__.__files__' ) )\
                   .filter( Request.JobID.in_( jobIDs ) ).all()

      reqDict['Successful'] = dict( ( jobId, reqObj ) for jobId, reqObj in ret )

      reqDict['Failed'] = dict( ( jobid, 'Request not found' ) for jobid in jobIDs - set( reqDict['Successful'] ) )
      session.expunge_all()
    except Exception as e:
      self.log.exception( "readRequestsForJobs: unexpected exception", lException = e )
      return S_ERROR( "readRequestsForJobs: unexpected exception : %s" % e )
    finally:
      session.close()

    return S_OK( reqDict )

Example 40

Project: floof Source File: authn.py
Function: check_certificate
    def check_certificate(self, request):
        """Check a client certificate serial and add authentication if valid."""

        self.state.pop('cert_serial', None)
        serial = get_certificate_serial(request)

        if not serial:
            # No cert. Our obligation to wipe cert state is fulfilled above.
            return

        # Figure out what certificate and user this serial belongs to
        # TODO: Optimize eagerloading
        serial = serial.lower()
        q = model.session.query(model.Certificate) \
            .options(joinedload_all('user.roles')) \
            .filter_by(serial=serial)

        try:
            cert = q.one()
        except NoResultFound:
            raise CertNotFoundError

        if cert.user.cert_auth == u'disabled':
            raise CertAuthDisabledError
        if cert.expired:
            raise CertExpiredError
        if cert.revoked:
            raise CertRevokedError
        if self.user and self.user.id != cert.user_id:
            raise AuthConflictError

        # At this point, we're confident that the supplied cert is valid

        self.state['cert_serial'] = serial
        self.trust.append('cert')

        if not self.user:
            self.user = cert.user

Example 41

Project: floof Source File: authn.py
Function: check_openid
    def check_openid(self, config):
        """Check OpenID state and add authentication if valid, else clear."""

        url = self.state.pop('openid_url', None)
        timestamp = self.state.pop('openid_timestamp', None)

        if not url or timestamp is None:
            # No (or corrupted) OpenID login. By popping, our obligation to
            # clear relevent state is already fulfilled, so just return
            return

        # TODO: Optimize eagerloading
        q = model.session.query(model.IdentityURL) \
            .options(joinedload_all('user.roles')) \
            .filter_by(url=url)

        try:
            openid = q.one()
        except NoResultFound:
            raise OpenIDNotFoundError

        if openid.user.cert_auth == 'required':
            raise OpenIDAuthDisabledError
        if self.user and self.user.id != openid.user_id:
            raise AuthConflictError

        # XXX Check timestamp sanity?
        # At this point, we're confident that the stored OpenID login is valid

        self.state['openid_url'] = url
        self.state['openid_timestamp'] = timestamp
        self.trust.append('openid')

        # Evaluate OpenID freshness
        confidence_expiry_secs = int(config.get(
            'auth.openid.expiry_seconds',
            DEFAULT_CONFIDENCE_EXPIRY))

        age = datetime.utcnow() - datetime.utcfromtimestamp(timestamp)
        if age <= timedelta(seconds=confidence_expiry_secs):
            self.trust.append('openid_recent')

        if not self.user:
            self.user = openid.user

Example 42

Project: floof Source File: authn.py
    def check_persona(self, config):
        """Check Persona state and add authentication if valid, else
        clear."""
        # XXX this is very similar to check_openid() above

        addr = self.state.pop('persona_addr', None)
        timestamp = self.state.pop('persona_timestamp', None)

        if not addr or not timestamp:
            # No (or corrupted) Persona login. By popping, our obligation to
            # clear relevent state is already fulfilled, so just return
            return

        # TODO: Optimize eagerloading
        q = model.session.query(model.IdentityEmail) \
            .options(joinedload_all('user.roles')) \
            .filter_by(email=addr)

        try:
            persona = q.one()
        except NoResultFound:
            raise PersonaNotFoundError

        if persona.user.cert_auth == 'required':
            raise PersonaAuthDisabledError
        if self.user and self.user.id != persona.user.id:
            raise AuthConflictError

        # At this point, we're confident that the stored Persona login is valid

        self.state['persona_addr'] = addr
        self.state['persona_timestamp'] = timestamp
        self.trust.append('persona')

        # Evaluate Persona freshness
        confidence_expiry_secs = int(config.get(
            'auth.persona.expiry_seconds',
            DEFAULT_CONFIDENCE_EXPIRY))

        age = datetime.utcnow() - datetime.utcfromtimestamp(timestamp)
        if age <= timedelta(seconds=confidence_expiry_secs):
            self.trust.append('persona_recent')

        if not self.user:
            self.user = persona.user

Example 43

Project: Flexget Source File: perf_tests.py
def imdb_query(session):
    import time
    from flexget.plugins.metainfo.imdb_lookup import Movie
    from flexget.plugins.cli.performance import log_query_count
    from sqlalchemy.sql.expression import select
    from progressbar import ProgressBar, Percentage, Bar, ETA
    from sqlalchemy.orm import joinedload_all

    imdb_urls = []

    log.info('Getting imdb_urls ...')
    # query so that we avoid loading whole object (maybe cached?)
    for _, url in session.execute(select([Movie.id, Movie.url])):
        imdb_urls.append(url)
    log.info('Got %i urls from database' % len(imdb_urls))
    if not imdb_urls:
        log.info('so .. aborting')
        return

    # commence testing

    widgets = ['Benchmarking - ', ETA(), ' ', Percentage(), ' ', Bar(left='[', right=']')]
    bar = ProgressBar(widgets=widgets, maxval=len(imdb_urls)).start()

    log_query_count('test')
    start_time = time.time()
    for index, url in enumerate(imdb_urls):
        bar.update(index)

        # movie = session.query(Movie).filter(Movie.url == url).first()
        # movie = session.query(Movie).options(subqueryload(Movie.genres)).filter(Movie.url == url).one()

        movie = session.query(Movie). \
            options(joinedload_all(Movie.genres, Movie.languages,
                                   Movie.actors, Movie.directors)). \
            filter(Movie.url == url).first()

        # access it's members so they're loaded
        [x.name for x in movie.genres]
        [x.name for x in movie.directors]
        [x.name for x in movie.actors]
        [x.name for x in movie.languages]

    log_query_count('test')
    took = time.time() - start_time
    log.debug('Took %.2f seconds to query %i movies' % (took, len(imdb_urls)))

Example 44

Project: kingbird Source File: api.py
Function: model_query
def model_query(context, *args):
    with read_session() as session:
        query = session.query(*args).options(joinedload_all('*'))
        return query