Here are the examples of the python api sqlalchemy.exc.ProgrammingError taken from open source projects. By voting up you can indicate which examples are most useful and appropriate.
31 Examples
3
Source : test_dialect.py
with Apache License 2.0
from gethue
with Apache License 2.0
from gethue
def test_autocommit_isolation_level(self):
c = testing.db.connect().execution_options(
isolation_level="AUTOCOMMIT"
)
# If we're really in autocommit mode then we'll get an error saying
# that the prepared transaction doesn't exist. Otherwise, we'd
# get an error saying that the command can't be run within a
# transaction.
assert_raises_message(
exc.ProgrammingError,
'prepared transaction with identifier "gilberte" does not exist',
c.execute,
"commit prepared 'gilberte'",
)
@testing.fails_on(
3
Source : apply.py
with Apache License 2.0
from Kraken-CI
with Apache License 2.0
from Kraken-CI
def _get_db_version(app):
with app.app_context():
db_version = None
try:
db_version = models.AlembicVersion.query.one_or_none()
if db_version is not None:
db_version = db_version.version_num
except sqlalchemy.exc.ProgrammingError as ex:
if not isinstance(ex.orig, psycopg2.errors.UndefinedTable):
raise
return db_version
def main():
3
Source : utils.py
with MIT License
from libercapital
with MIT License
from libercapital
def create_db():
try:
with settings.ENGINE_NO_DB.connect() as connection:
sql = f"CREATE DATABASE {settings.POSTGRES_DB};"
print(f"CREATING DATABASE: ['{sql}']", end='...', flush=True)
connection.connection.set_isolation_level(0)
connection.execute(sql)
connection.connection.set_isolation_level(1)
print('Done!')
except sqlalchemy.exc.ProgrammingError:
print('database already exists... skipping', end='... ')
print('Done!')
def create_or_drop_all_tables(cmd, dict_db_models=dict_db_models):
3
Source : utils.py
with MIT License
from libercapital
with MIT License
from libercapital
def create_or_drop_all_tables(cmd, dict_db_models=dict_db_models):
print(f'[{cmd.upper()} ALL TABLES]')
for e, table_name in enumerate(dict_db_models.keys(), 1):
table_model = dict_db_models[table_name]
print(f'[{e}/{len(dict_db_models.keys())}] {cmd} table ->',
dict_db_models[table_name].__tablename__,
end='...', flush=True)
_method = getattr(table_model.__table__, cmd)
try:
_method(bind=settings.ENGINE)
except sqlalchemy.exc.ProgrammingError:
print('skipping... ', end='... ')
print('Done!')
def phoenix():
3
Source : info.py
with MIT License
from refraction-ray
with MIT License
from refraction-ray
def _fetch_sql(self, path):
"""
fetch the information and pricetable from sql, not recommend to use manually,
just set the fetch label to be true when init the object
:param path: engine object from sqlalchemy
"""
try:
pricetable = pd.read_sql("xa" + self.code, path)
self.price = pricetable
except exc.ProgrammingError as e:
# print('no saved copy of %s' % self.code)
raise e
def update(self):
3
Source : info.py
with MIT License
from refraction-ray
with MIT License
from refraction-ray
def _fetch_sql(self, path):
"""
fetch the information and pricetable from sql, not recommend to use manually,
just set the fetch label to be true when init the object
:param path: engine object from sqlalchemy
"""
try:
content = pd.read_sql("xa" + self.code, path)
pricetable = content.iloc[1:]
commentl = [float(com) for com in pricetable.comment]
self.price = pricetable[["date", "netvalue", "totvalue"]]
self.price["comment"] = commentl
self.name = json.loads(content.iloc[0].comment)["name"]
except exc.ProgrammingError as e:
# print('no saved copy of %s' % self.code)
raise e
def update(self):
3
Source : test_generic.py
with MIT License
from schireson
with MIT License
from schireson
def test_implicit_schema(self, pg_implicit_schema):
pg_implicit_schema.execute("select * from report")
with pytest.raises(sqlalchemy.exc.ProgrammingError):
pg_implicit_schema.execute("select * from public.quarter")
def test_explicit_schema(self, pg_explicit_schema):
3
Source : test_generic.py
with MIT License
from schireson
with MIT License
from schireson
def test_explicit_schema(self, pg_explicit_schema):
pg_explicit_schema.execute("select * from quarter")
with pytest.raises(sqlalchemy.exc.ProgrammingError):
pg_explicit_schema.execute("select * from report")
rows = Rows(Quarter(id=1, year=1, quarter=1))
3
Source : test_dialect.py
with MIT License
from sqlalchemy
with MIT License
from sqlalchemy
def test_autocommit_isolation_level(self):
c = testing.db.connect().execution_options(
isolation_level="AUTOCOMMIT"
)
# If we're really in autocommit mode then we'll get an error saying
# that the prepared transaction doesn't exist. Otherwise, we'd
# get an error saying that the command can't be run within a
# transaction.
assert_raises_message(
exc.ProgrammingError,
'prepared transaction with identifier "gilberte" does not exist',
c.exec_driver_sql,
"commit prepared 'gilberte'",
)
def test_extract(self, connection):
0
Source : cockroach.py
with Apache License 2.0
from dcos
with Apache License 2.0
from dcos
def _wait_for_cockroach_start(self):
log.info('Waiting for CockroachDB to start')
def _accepting_connections():
_engine = sqlalchemy.create_engine(
self.sql_alchemy_url_for_db('iam'),
echo=False,
connect_args={}
)
session_factory = sqlalchemy.orm.sessionmaker(bind=_engine)
session = session_factory(autocommit=True)
try:
session.execute('show tables;')
return True
except sqlalchemy.exc.ProgrammingError as exc:
if hasattr(exc.orig, 'pgcode'):
# This works with CockroachDB.
if exc.orig.pgcode == psycopg2.errorcodes.INVALID_CATALOG_NAME:
log.info('Database `iam` does not exist.')
return True
except sqlalchemy.exc.OperationalError:
# Connection failed
return False
finally:
session.close()
while True:
if _accepting_connections():
log.info('CockroachDB accepts connections')
return
time.sleep(0.2)
0
Source : test_mssql_lms_operations.py
with Apache License 2.0
from Ed-Fi-Alliance-OSS
with Apache License 2.0
from Ed-Fi-Alliance-OSS
def describe_when_getting_processed_files():
def describe_given_parameters_are_correct():
def it_should_build_the_correct_sql_query(mocker):
resource_name = "fake_resource_name"
expected_query = """
SELECT
FullPath
FROM
lms.ProcessedFiles
WHERE
ResourceName = 'fake_resource_name'
""".strip()
query_mock = mocker.patch.object(
pd, "read_sql_query", return_value=pd.DataFrame(columns=["FullPath"])
)
SqlLmsOperations(Mock()).get_processed_files(resource_name)
call_args = query_mock.call_args
assert len(call_args) == 2
assert call_args[0][0] == expected_query
def describe_given_the_processed_file_table_does_not_exist():
def it_should_log_an_error_and_re_raise_the_exception(mocker):
def __raise(query, engine) -> None:
raise ProgrammingError("statement", [], "none", False)
mocker.patch.object(pd, "read_sql_query", side_effect=__raise)
# Typically we do not test the logging. In this case and another
# below, logging was a core business requirement for a task, and
# thus it makes sense to test that requirement.
logger = logging.getLogger("edfi_lms_ds_loader.sql_lms_operations")
mock_exc_logger = mocker.patch.object(logger, "exception")
with pytest.raises(ProgrammingError):
SqlLmsOperations(Mock()).get_processed_files("fake_resource_name")
mock_exc_logger.assert_called_once()
def describe_when_adding_processed_files():
0
Source : test_mssql_lms_operations.py
with Apache License 2.0
from Ed-Fi-Alliance-OSS
with Apache License 2.0
from Ed-Fi-Alliance-OSS
def describe_when_adding_processed_files():
def describe_given_parameters_are_correct():
def it_should_build_the_correct_sql_statement(mocker):
resource_name = "fake_resource_name"
path = "fake_path/"
rows = 3
expected_statement = """
INSERT INTO
lms.ProcessedFiles
(
FullPath,
ResourceName,
NumberOfRows
)
VALUES
(
'fake_path/',
'fake_resource_name',
3
)
""".strip()
exec_mock = mocker.patch.object(SqlLmsOperations, "_exec")
SqlLmsOperations(Mock()).add_processed_file(path, resource_name, rows)
exec_mock.assert_called_with(expected_statement)
def describe_given_the_processed_file_table_does_not_exist():
def it_should_log_an_error_and_re_raise_the_exception(mocker):
def __raise(_) -> None:
raise ProgrammingError("statement", [], "none", False)
resource_name = "fake_resource_name"
path = "fake_path/"
rows = 3
mocker.patch.object(SqlLmsOperations, "_exec", side_effect=__raise)
logger = logging.getLogger("edfi_lms_ds_loader.sql_lms_operations")
mock_exc_logger = mocker.patch.object(logger, "exception")
with pytest.raises(ProgrammingError):
SqlLmsOperations(Mock()).add_processed_file(path, resource_name, rows)
mock_exc_logger.assert_called_once()
# insert_new_records_to_production_for_attendance_events
0
Source : schedulers.py
with GNU General Public License v2.0
from flathub
with GNU General Public License v2.0
from flathub
def classifyChanges(self, schedulerid, classifications):
def thd(conn):
tbl = self.db.model.scheduler_changes
ins_q = tbl.insert()
upd_q = tbl.update(
((tbl.c.schedulerid == schedulerid) &
(tbl.c.changeid == sa.bindparam('wc_changeid'))))
for changeid, important in classifications.items():
transaction = conn.begin()
# convert the 'important' value into an integer, since that
# is the column type
imp_int = int(bool(important))
try:
conn.execute(ins_q,
schedulerid=schedulerid,
changeid=changeid,
important=imp_int).close()
except (sqlalchemy.exc.ProgrammingError,
sqlalchemy.exc.IntegrityError):
transaction.rollback()
transaction = conn.begin()
# insert failed, so try an update
conn.execute(upd_q,
wc_changeid=changeid,
important=imp_int).close()
transaction.commit()
return self.db.pool.do(thd)
# returns a Deferred that returns None
def flushChangeClassifications(self, schedulerid, less_than=None):
0
Source : test_on_conflict.py
with Apache License 2.0
from gethue
with Apache License 2.0
from gethue
def test_on_conflict_do_update_exotic_targets_five(self):
users = self.tables.users_xtra
with testing.db.connect() as conn:
self._exotic_targets_fixture(conn)
# try bogus index
i = insert(users)
i = i.on_conflict_do_update(
index_elements=self.bogus_index.columns,
index_where=self.bogus_index.dialect_options["postgresql"][
"where"
],
set_=dict(
name=i.excluded.name, login_email=i.excluded.login_email
),
)
assert_raises(
exc.ProgrammingError,
conn.execute,
i,
dict(
id=1,
name="namebogus",
login_email="[email protected]",
lets_index_this="bogus",
),
)
def test_on_conflict_do_update_exotic_targets_six(self):
0
Source : test_types.py
with Apache License 2.0
from gethue
with Apache License 2.0
from gethue
def test_remain_on_table_metadata_wide(self):
metadata = self.metadata
e1 = Enum("one", "two", "three", name="myenum", metadata=metadata)
table = Table("e1", metadata, Column("c1", e1))
# need checkfirst here, otherwise enum will not be created
assert_raises_message(
sa.exc.ProgrammingError,
'.*type "myenum" does not exist',
table.create,
)
table.create(checkfirst=True)
table.drop()
table.create(checkfirst=True)
table.drop()
assert "myenum" in [e["name"] for e in inspect(testing.db).get_enums()]
metadata.drop_all()
assert "myenum" not in [
e["name"] for e in inspect(testing.db).get_enums()
]
def test_non_native_dialect(self):
0
Source : test_execute.py
with Apache License 2.0
from gethue
with Apache License 2.0
from gethue
def test_handle_error_event_revalidate(self):
dbapi = self.dbapi
class MySpecialException(Exception):
pass
eng = create_engine("sqlite://", module=dbapi, _initialize=False)
@event.listens_for(eng, "handle_error")
def handle_error(ctx):
assert ctx.engine is eng
assert ctx.connection is conn
assert isinstance(
ctx.sqlalchemy_exception, tsa.exc.ProgrammingError
)
raise MySpecialException("failed operation")
conn = eng.connect()
conn.invalidate()
dbapi.connect = Mock(side_effect=self.ProgrammingError("random error"))
assert_raises(MySpecialException, getattr, conn, "connection")
def test_handle_error_event_implicit_revalidate(self):
0
Source : test_execute.py
with Apache License 2.0
from gethue
with Apache License 2.0
from gethue
def test_handle_error_event_implicit_revalidate(self):
dbapi = self.dbapi
class MySpecialException(Exception):
pass
eng = create_engine("sqlite://", module=dbapi, _initialize=False)
@event.listens_for(eng, "handle_error")
def handle_error(ctx):
assert ctx.engine is eng
assert ctx.connection is conn
assert isinstance(
ctx.sqlalchemy_exception, tsa.exc.ProgrammingError
)
raise MySpecialException("failed operation")
conn = eng.connect()
conn.invalidate()
dbapi.connect = Mock(side_effect=self.ProgrammingError("random error"))
assert_raises(MySpecialException, conn.execute, select([1]))
def test_handle_error_custom_connect(self):
0
Source : connection.py
with MIT License
from mgao6767
with MIT License
from mgao6767
def raw_sql(
self, sql, coerce_float=True, date_cols=None, index_col=None, params=None
):
"""
Queries the database using a raw SQL string.
:param sql: SQL code in string object.
:param coerce_float: (optional) boolean, default: True
Attempt to convert values to non-string, non-numeric objects
to floating point. Can result in loss of precision.
:param date_cols: (optional) list or dict, default: None
- List of column names to parse as date
- Dict of ``{column_name: format string}`` where
format string is:
strftime compatible in case of parsing string times or
is one of (D, s, ns, ms, us) in case of parsing
integer timestamps
- Dict of ``{column_name: arg dict}``,
where the arg dict corresponds to the keyword arguments of
:func:`pandas.to_datetime`
:param index_col: (optional) string or list of strings,
default: None
Column(s) to set as index(MultiIndex)
:param params: parameters to SQL query, if parameterized.
:rtype: pandas.DataFrame
Usage ::
# Basic Usage
>>> data = db.raw_sql('select cik, fdate, coname from wrdssec_all.dforms;', date_cols=['fdate'], index_col='cik')
>>> data.head()
cik fdate coname
0000000003 1995-02-15 DEFINED ASSET FUNDS MUNICIPAL INVT TR FD NEW Y...
0000000003 1996-02-14 DEFINED ASSET FUNDS MUNICIPAL INVT TR FD NEW Y...
0000000003 1997-02-19 DEFINED ASSET FUNDS MUNICIPAL INVT TR FD NEW Y...
0000000003 1998-03-02 DEFINED ASSET FUNDS MUNICIPAL INVT TR FD NEW Y...
0000000003 1998-03-10 DEFINED ASSET FUNDS MUNICIPAL INVT TR FD NEW Y..
...
# Parameterized SQL query
>>> parm = {'syms': ('A', 'AA', 'AAPL'), 'num_shares': 50000}
>>> data = db.raw_sql('select * from taqmsec.ctm_20030910 where sym_root in %(syms)s and size > %(num_shares)s', params=parm)
>>> data.head()
date time_m ex sym_root sym_suffix tr_scond size price tr_stopind tr_corr tr_seqnum tr_source tr_rf
2003-09-10 11:02:09.485000 T A None None 211400.0 25.350 N 00 1.929952e+15 C None
2003-09-10 11:04:29.508000 N A None None 55500.0 25.180 N 00 1.929952e+15 C None
2003-09-10 15:08:21.155000 N A None None 50500.0 24.470 N 00 1.929967e+15 C None
2003-09-10 16:10:35.522000 T A None B 71900.0 24.918 N 00 1.929970e+15 C None
2003-09-10 09:35:20.709000 N AA None None 108100.0 28.200 N 00 1.929947e+15 C None
"""
try:
return pd.read_sql_query(
sql,
self.connection,
coerce_float=coerce_float,
parse_dates=date_cols,
index_col=index_col,
params=params,
)
except sa.exc.ProgrammingError as e:
raise e
def get_table(
0
Source : db.py
with GNU General Public License v3.0
from OpenSIPS
with GNU General Public License v3.0
from OpenSIPS
def ensure_user(self, db_url):
url = make_url(db_url)
if url.password is None:
logger.error("database URL does not include a password")
return False
if url.drivername.lower() == "mysql":
sqlcmd = "CREATE USER IF NOT EXISTS '{}' IDENTIFIED BY '{}'".format(
url.username, url.password)
try:
result = self.__conn.execute(sqlcmd)
if result:
logger.info("created user '%s'", url.username)
except:
logger.error("failed to create user '%s'", url.username)
return False
if url.username == 'root':
logger.debug("skipping password change for root user")
else:
"""
Query compatibility facts when changing a MySQL user password:
- SET PASSWORD syntax has diverged between MySQL and MariaDB
- ALTER USER syntax is not supported in MariaDB < 10.2
"""
# try MariaDB syntax first
sqlcmd = "SET PASSWORD FOR '{}' = PASSWORD('{}')".format(
url.username, url.password)
try:
result = self.__conn.execute(sqlcmd)
if result:
logger.info("set password '%s%s%s' for '%s' (MariaDB)",
url.password[0] if len(url.password) >= 1 else '',
(len(url.password) - 2) * '*',
url.password[-1] if len(url.password) >= 2 else '',
url.username)
except sqlalchemy.exc.ProgrammingError as se:
try:
if int(se.args[0].split(",")[0].split("(")[2]) == 1064:
# syntax error! OK, now try Oracle MySQL syntax
sqlcmd = "ALTER USER '{}' IDENTIFIED BY '{}'".format(
url.username, url.password)
result = self.__conn.execute(sqlcmd)
if result:
logger.info("set password '%s%s%s' for '%s' (MySQL)",
url.password[0] if len(url.password) >= 1 else '',
(len(url.password) - 2) * '*',
url.password[-1] if len(url.password) >= 2 else '',
url.username)
except:
logger.exception("failed to set password for '%s'", url.username)
return False
except:
logger.exception("failed to set password for '%s'", url.username)
return False
sqlcmd = "GRANT ALL ON {}.* TO '{}'".format(self.db_name, url.username)
try:
result = self.__conn.execute(sqlcmd)
if result:
logger.info("granted access to user '%s' on DB '%s'",
url.username, self.db_name)
except:
logger.exception("failed to grant access to '%s' on DB '%s'",
url.username, self.db_name)
return False
sqlcmd = "FLUSH PRIVILEGES"
try:
result = self.__conn.execute(sqlcmd)
logger.info("flushed privileges")
except:
logger.exception("failed to flush privileges")
return False
elif url.drivername.lower() == "postgres":
if not self.exists_role(url.username):
logger.info("creating role %s", url.username)
if not self.create_role(url.username, url.password):
logger.error("failed to create role %s", url.username)
self.create_role(url.username, url.password, update=True)
sqlcmd = "GRANT ALL PRIVILEGES ON DATABASE {} TO {}".format(
self.db_name, url.username)
logger.info(sqlcmd)
try:
result = self.__conn.execute(sqlcmd)
if result:
logger.debug("... OK")
except:
logger.error("failed to grant ALL to '%s' on db '%s'",
url.username, self.db_name)
return False
return True
def create_role(self, role_name, role_password, update=False,
0
Source : info.py
with MIT License
from refraction-ray
with MIT License
from refraction-ray
def __init__(
self,
code,
fetch=False,
save=False,
path="",
form="csv",
round_label=0,
dividend_label=0,
value_label=0,
):
# 增量 IO 的逻辑都由 basicinfo 类来处理,对于具体的子类,只需实现_save_form 和 _fetch_form 以及 update 函数即可
self.code = code
self.round_label = round_label
self.dividend_label = dividend_label
self.value_label = value_label
self.specialdate = []
self.fenhongdate = []
self.zhesuandate = []
# compatible with new ``xa.set_backend()`` API
import xalpha.universal as xu
if (xu.ioconf["backend"] in ["csv", "sql"]) and (not path):
fetch = True
save = True
form = xu.ioconf["backend"]
path = xu.ioconf["path"]
if xu.ioconf["backend"] == "csv":
path = os.path.join(path, xu.ioconf["prefix"] + "INFO-")
self.format = form
if fetch is False:
self._basic_init() # update self. name rate and price table
else:
try:
self.fetch(path, self.format)
df = self.update() # update the price table as well as the file
if (df is not None) and save is True:
self.save(path, self.format, option="a", delta=df)
except (FileNotFoundError, exc.ProgrammingError, exc.OperationalError) as e:
logger.info("no saved copy of %s" % self.code)
fetch = False
self._basic_init()
if (save is True) and (fetch is False):
self.save(path, self.format)
def _basic_init(self):
0
Source : info.py
with MIT License
from refraction-ray
with MIT License
from refraction-ray
def _fetch_sql(self, path):
"""
fetch the information and pricetable from sql, not recommend to use manually,
just set the fetch label to be true when init the object
:param path: engine object from sqlalchemy
"""
try:
content = pd.read_sql("xa" + self.code, path)
pricetable = content.iloc[1:]
commentl = [float(com) for com in pricetable.comment]
self.price = pricetable[["date", "netvalue", "totvalue"]]
self.price["comment"] = commentl
saveinfo = json.loads(content.iloc[0].comment)
if not isinstance(saveinfo, dict):
raise FundTypeError("This csv doesn't looks like from fundinfo")
self.segment = saveinfo["segment"]
self.feeinfo = saveinfo["feeinfo"]
self.name = saveinfo["name"]
self.rate = saveinfo["rate"]
except exc.ProgrammingError as e:
# print('no saved copy of %s' % self.code)
raise e
def _hk_update(self):
0
Source : universal.py
with MIT License
from refraction-ray
with MIT License
from refraction-ray
def cachedio(**ioconf):
"""
用法类似:func:`cached`,通用透明缓存器,用来作为 (code, start, end ...) -> pd.DataFrame 形式函数的缓存层,
避免重复爬取已有数据。
:param **ioconf: 可选关键字参数 backend: csv or sql or memory,
path: csv 文件夹或 sql engine, refresh True 会刷新结果,重新爬取, default False,
prefix 是 key 前统一部分, 缓存 hash 标志
:return:
"""
def cached(f):
@wraps(f)
def wrapper(*args, **kws):
if args:
code = args[0]
else:
code = kws.get("code")
date = ioconf.get("date", "date") # 没利用上这个栏的名字变化
precached = ioconf.get("precached", None)
precached = kws.get("precached", precached)
key = kws.get("key", code)
key = key.replace("/", " ")
key_func = ioconf.get("key_func", None)
key_func = ioconf.get("keyfunc", key_func)
if key_func is not None:
key = key_func(key)
defaultend = ioconf.get("defaultend", today_obj)
defaultend = ioconf.get("default_end", defaultend)
defaultprev = ioconf.get("defaultprev", 365)
defaultprev = ioconf.get("default_prev", defaultprev)
if isinstance(defaultend, str):
defaultend = defaultend.replace("/", "").replace("-", "")
defaultend = dt.datetime.strptime(defaultend, "%Y%m%d")
if callable(defaultend):
defaultend = defaultend()
start = kws.get("start", None)
end = kws.get("end", None)
prev = kws.get("prev", None)
prefix = ioconf.get("prefix", "")
key = prefix + key
if precached:
precached = precached.replace("/", "").replace("-", "")
precached_obj = dt.datetime.strptime(precached, "%Y%m%d")
if not prev:
prev = defaultprev
if not end:
end_obj = defaultend
else:
end_obj = dt.datetime.strptime(
end.replace("/", "").replace("-", ""), "%Y%m%d"
)
if not start:
start_obj = end_obj - dt.timedelta(days=prev)
else:
start_obj = dt.datetime.strptime(
start.replace("/", "").replace("-", ""), "%Y%m%d"
)
start_str = start_obj.strftime("%Y%m%d")
end_str = end_obj.strftime("%Y%m%d")
backend = ioconf.get("backend")
backend = kws.get("backend", backend)
# if backend == "sql": # reserved for case insensitive database settings
# key = key.lower()
refresh = ioconf.get("refresh", False)
refresh = kws.get("refresh", refresh)
fetchonly = ioconf.get("fetchonly", False)
fetchonly = ioconf.get("fetch_only", fetchonly)
fetchonly = kws.get("fetchonly", fetchonly)
fetchonly = kws.get("fetch_only", fetchonly)
path = ioconf.get("path")
path = kws.get("path", path)
kws["start"] = start_str
kws["end"] = end_str
if not backend:
df = f(*args, **kws)
df = df[df["date"] < = kws["end"]]
df = df[df["date"] >= kws["start"]]
return df
else:
if backend == "csv":
key = key + ".csv"
if not getattr(thismodule, "cached_dict", None):
setattr(thismodule, "cached_dict", {})
if refresh:
is_changed = True
df0 = f(*args, **kws)
else: # non refresh
try:
if backend == "csv":
if key in getattr(thismodule, "cached_dict"):
# 即使硬盘级别的缓存,也有内存层,加快读写速度
df0 = getattr(thismodule, "cached_dict")[key]
else:
df0 = pd.read_csv(os.path.join(path, key))
elif backend == "sql":
if key in getattr(thismodule, "cached_dict"):
df0 = getattr(thismodule, "cached_dict")[key]
else:
df0 = pd.read_sql(key, path)
elif backend == "memory":
df0 = getattr(thismodule, "cached_dict")[key]
else:
raise ValueError("no %s option for backend" % backend)
df0[date] = pd.to_datetime(df0[date])
# 向前延拓
is_changed = False
if df0.iloc[0][date] > start_obj and not fetchonly:
kws["start"] = start_str
kws["end"] = (
df0.iloc[0][date] - pd.Timedelta(days=1)
).strftime("%Y%m%d")
if has_weekday(kws["start"], kws["end"]):
# 考虑到海外市场的不同情况,不用 opendate 判断,采取保守型判别
df1 = f(*args, **kws)
if df1 is not None and len(df1) > 0:
df1 = df1[df1["date"] < = kws["end"]]
if df1 is not None and len(df1) > 0:
is_changed = True
df0 = df1.append(df0, ignore_index=True, sort=False)
# 向后延拓
if df0.iloc[-1][date] < end_obj and not fetchonly:
nextday_str = (
df0.iloc[-1][date] + dt.timedelta(days=1)
).strftime("%Y%m%d")
if len(df0[df0["date"] == df0.iloc[-1]["date"]]) == 1:
kws["start"] = (df0.iloc[-1][date]).strftime("%Y%m%d")
else: # 单日多行的表默认最后一日是准确的,不再刷新了
kws["start"] = nextday_str
kws["end"] = end_str
if has_weekday(nextday_str, kws["end"]): # 新更新的日期里有工作日
df2 = f(*args, **kws)
if df2 is not None and len(df2) > 0:
df2 = df2[df2["date"] >= kws["start"]]
if df2 is not None and len(df2) > 0:
is_changed = True
if (
len(df0[df0["date"] == df0.iloc[-1]["date"]])
== 1
):
df0 = df0.iloc[:-1]
df0 = df0.append(df2, ignore_index=True, sort=False)
# 注意这里抹去更新了原有最后一天的缓存,这是因为日线最新一天可能有实时数据污染
except (FileNotFoundError, exc.ProgrammingError, KeyError) as e:
if fetchonly:
logger.error(
"no cache in backend for %s but you insist `fetchonly`"
% code
)
raise e
if precached:
if start_obj > precached_obj:
kws["start"] = precached
if end_obj < today_obj():
kws["end"] = (
today_obj() - dt.timedelta(days=1)
).strftime("%Y%m%d")
is_changed = True
df0 = f(*args, **kws)
if df0 is not None and len(df0) > 0 and is_changed:
if backend == "csv":
df0.to_csv(os.path.join(path, key), index=False)
elif backend == "sql":
df0.to_sql(key, con=path, if_exists="replace", index=False)
# elif backend == "memory":
# 总是刷新内存层,即使是硬盘缓存
d = getattr(thismodule, "cached_dict")
d[key] = df0
if df0 is not None and len(df0) > 0:
df0 = df0[df0["date"] < = end_str]
df0 = df0[df0["date"] >= start_str]
return df0
return wrapper
return cached
def fetch_backend(key):
0
Source : universal.py
with MIT License
from refraction-ray
with MIT License
from refraction-ray
def fetch_backend(key):
prefix = ioconf.get("prefix", "")
key = prefix + key
backend = ioconf.get("backend")
path = ioconf.get("path")
if backend == "csv":
key = key + ".csv"
try:
if backend == "csv":
df0 = pd.read_csv(os.path.join(path, key))
elif backend == "sql":
df0 = pd.read_sql(key, path)
else:
raise ValueError("no %s option for backend" % backend)
return df0
except (FileNotFoundError, exc.ProgrammingError, KeyError):
return None
def save_backend(key, df, mode="a", header=False):
0
Source : test_patch.py
with MIT License
from schireson
with MIT License
from schireson
def test_tightly_scoped_patch(redshift, postgres):
"""Assert psycopg2's patch is tightly scoped
Redshift combined with a 2nd non-redshift fixture in the same test should not
add redshift-specific features to the other engine.
"""
import moto
copy_command = COPY_TEMPLATE.format(
COMMAND="COPY",
LOCATION="s3://mybucket/file.csv",
COLUMNS="",
FROM="from",
CREDENTIALS="credentials",
OPTIONAL_ARGS="",
)
with moto.mock_s3():
setup_table_and_bucket(redshift)
setup_table_and_bucket(postgres, create_bucket=False)
redshift.execute(copy_command)
with pytest.raises(sqlalchemy.exc.ProgrammingError) as e:
postgres.execute(copy_command)
assert 'syntax error at or near "credentials"' in str(e.value)
0
Source : test_unload.py
with MIT License
from schireson
with MIT License
from schireson
def test_ignores_sqlalchmey_text_obj(redshift):
"""Test command ignores SQLAlchemy Text Objects and raises error."""
with moto.mock_s3():
setup_table_and_insert_data(redshift)
try:
redshift.execute(
text(
UNLOAD_TEMPLATE.format(
COMMAND=" UNLOAD",
SELECT_STATEMENT="select * from test_s3_unload_from_redshift",
TO="TO",
LOCATION="s3://mybucket/myfile.csv",
AUTHORIZATION="AUTHORIZATION",
OPTIONAL_ARGS="DELIMITER AS ','",
)
)
)
# The default engine will try to execute an `UNLOAD` command and will fail, raising a
# ProgrammingError
except sqlalchemy.exc.ProgrammingError:
return
def test_multiple_sql_statemts(redshift):
0
Source : test_users.py
with MIT License
from Sentimentron
with MIT License
from Sentimentron
def try_drop_existing_db(self):
"""
Drop the existing test database, if it exists.
:return: True on success
"""
conn = create_engine("postgresql+psycopg2://annotatron:annotatron@localhost:5432/postgres").connect()
conn = conn.execution_options(autocommit=False)
conn.execute("ROLLBACK")
try:
conn.execute("DROP DATABASE %s" % self.db_name)
except exc.ProgrammingError as e:
# Could not drop the database, probably does not exist
conn.execute("ROLLBACK")
logging.error("Could not drop the test database: %s", e)
except exc.OperationalError as e:
# Could not drop database because it's being accessed by other users (psql prompt open?)
conn.execute("ROLLBACK")
logging.fatal("Could not drop the test database, due to concurrent access.")
raise e
return True
@classmethod
0
Source : dev.py
with MIT License
from sharebears
with MIT License
from sharebears
def insertdata():
"""Recreate and repopulate an existing testing database."""
with app.test_request_context():
flask.g.cache_keys = defaultdict(set)
try:
user = User.from_pk(1)
except sqlalchemy.exc.ProgrammingError as e:
click.secho(
'Please manually create the database before running the createdata command.',
fg='red',
bold=True,
)
raise click.Abort
if user and user.username != 'user_one':
click.secho(
'The username of the first user is not user_one. Are you sure this '
'is a testing database? Please clear it yourself and re-try.',
fg='red',
bold=True,
)
raise click.Abort
cache.clear()
db.session.commit()
with app.app_context():
db.drop_all()
db.create_all()
with app.test_request_context():
for p in POPULATORS:
p.populate()
CorePopulator.add_permissions(SitePermissions.GOD_MODE)
cache.clear()
click.echo(f'Updated and inserted development data into the database!')
@dev.command()
0
Source : test_on_conflict.py
with MIT License
from sqlalchemy
with MIT License
from sqlalchemy
def test_on_conflict_do_update_exotic_targets_five(self, connection):
users = self.tables.users_xtra
self._exotic_targets_fixture(connection)
# try bogus index
i = insert(users)
i = i.on_conflict_do_update(
index_elements=self.bogus_index.columns,
index_where=self.bogus_index.dialect_options["postgresql"][
"where"
],
set_=dict(
name=i.excluded.name, login_email=i.excluded.login_email
),
)
assert_raises(
exc.ProgrammingError,
connection.execute,
i,
dict(
id=1,
name="namebogus",
login_email="[email protected]",
lets_index_this="bogus",
),
)
def test_on_conflict_do_update_exotic_targets_six(self, connection):
0
Source : test_types.py
with MIT License
from sqlalchemy
with MIT License
from sqlalchemy
def test_create_drop_schema_translate_map(self, connection):
conn = connection.execution_options(
schema_translate_map={None: testing.config.test_schema}
)
e1 = Enum("one", "two", "three", name="myenum")
assert "myenum" not in [
e["name"]
for e in inspect(connection).get_enums(testing.config.test_schema)
]
e1.create(conn, checkfirst=True)
e1.create(conn, checkfirst=True)
assert "myenum" in [
e["name"]
for e in inspect(connection).get_enums(testing.config.test_schema)
]
s1 = conn.begin_nested()
assert_raises(exc.ProgrammingError, e1.create, conn, checkfirst=False)
s1.rollback()
e1.drop(conn, checkfirst=True)
e1.drop(conn, checkfirst=True)
assert "myenum" not in [
e["name"]
for e in inspect(connection).get_enums(testing.config.test_schema)
]
assert_raises(exc.ProgrammingError, e1.drop, conn, checkfirst=False)
def test_remain_on_table_metadata_wide(self, metadata, future_connection):
0
Source : test_types.py
with MIT License
from sqlalchemy
with MIT License
from sqlalchemy
def test_remain_on_table_metadata_wide(self, metadata, future_connection):
connection = future_connection
e1 = Enum("one", "two", "three", name="myenum", metadata=metadata)
table = Table("e1", metadata, Column("c1", e1))
# need checkfirst here, otherwise enum will not be created
assert_raises_message(
sa.exc.ProgrammingError,
'.*type "myenum" does not exist',
table.create,
connection,
)
connection.rollback()
table.create(connection, checkfirst=True)
table.drop(connection)
table.create(connection, checkfirst=True)
table.drop(connection)
assert "myenum" in [e["name"] for e in inspect(connection).get_enums()]
metadata.drop_all(connection)
assert "myenum" not in [
e["name"] for e in inspect(connection).get_enums()
]
def test_non_native_dialect(self, metadata, testing_engine):
0
Source : test_execute.py
with MIT License
from sqlalchemy
with MIT License
from sqlalchemy
def test_handle_error_event_implicit_revalidate(self):
dbapi = self.dbapi
class MySpecialException(Exception):
pass
eng = create_engine("sqlite://", module=dbapi, _initialize=False)
@event.listens_for(eng, "handle_error")
def handle_error(ctx):
assert ctx.engine is eng
assert ctx.connection is conn
assert isinstance(
ctx.sqlalchemy_exception, tsa.exc.ProgrammingError
)
raise MySpecialException("failed operation")
conn = eng.connect()
conn.invalidate()
dbapi.connect = Mock(side_effect=self.ProgrammingError("random error"))
assert_raises(MySpecialException, conn.execute, select(1))
def test_handle_error_custom_connect(self):