generic_assets.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844
  1. from __future__ import annotations
  2. from datetime import datetime, timedelta
  3. from typing import Any
  4. import json
  5. from flask import current_app
  6. from flask_security import current_user
  7. import pandas as pd
  8. from sqlalchemy import select
  9. from sqlalchemy.engine import Row
  10. from sqlalchemy.ext.hybrid import hybrid_method
  11. from sqlalchemy.sql.expression import func, text
  12. from sqlalchemy.ext.mutable import MutableDict, MutableList
  13. from timely_beliefs import BeliefsDataFrame, utils as tb_utils
  14. from flexmeasures.data import db
  15. from flexmeasures.data.models.annotations import Annotation, to_annotation_frame
  16. from flexmeasures.data.models.charts import chart_type_to_chart_specs
  17. from flexmeasures.data.models.data_sources import DataSource
  18. from flexmeasures.data.models.parsing_utils import parse_source_arg
  19. from flexmeasures.data.models.user import User
  20. from flexmeasures.data.queries.annotations import query_asset_annotations
  21. from flexmeasures.data.services.timerange import get_timerange
  22. from flexmeasures.auth.policy import (
  23. AuthModelMixin,
  24. EVERY_LOGGED_IN_USER,
  25. ACCOUNT_ADMIN_ROLE,
  26. )
  27. from flexmeasures.utils import geo_utils
  28. from flexmeasures.utils.coding_utils import flatten_unique
  29. from flexmeasures.utils.time_utils import determine_minimum_resampling_resolution
  30. class GenericAssetType(db.Model):
  31. """An asset type defines what type an asset belongs to.
  32. Examples of asset types: WeatherStation, Market, CP, EVSE, WindTurbine, SolarPanel, Building.
  33. """
  34. id = db.Column(db.Integer, primary_key=True)
  35. name = db.Column(db.String(80), default="", unique=True)
  36. description = db.Column(db.String(80), nullable=True, unique=False)
  37. def __repr__(self):
  38. return "<GenericAssetType %s: %r>" % (self.id, self.name)
  39. class GenericAsset(db.Model, AuthModelMixin):
  40. """An asset is something that has economic value.
  41. Examples of tangible assets: a house, a ship, a weather station.
  42. Examples of intangible assets: a market, a country, a copyright.
  43. """
  44. __table_args__ = (
  45. db.CheckConstraint(
  46. "parent_asset_id != id", name="generic_asset_self_reference_ck"
  47. ),
  48. db.UniqueConstraint(
  49. "name",
  50. "parent_asset_id",
  51. name="generic_asset_name_parent_asset_id_key",
  52. ),
  53. )
  54. # No relationship
  55. id = db.Column(db.Integer, primary_key=True)
  56. name = db.Column(db.String(80), default="")
  57. latitude = db.Column(db.Float, nullable=True)
  58. longitude = db.Column(db.Float, nullable=True)
  59. attributes = db.Column(MutableDict.as_mutable(db.JSON), nullable=False, default={})
  60. sensors_to_show = db.Column(
  61. MutableList.as_mutable(db.JSON), nullable=False, default=[]
  62. )
  63. flex_context = db.Column(
  64. MutableDict.as_mutable(db.JSON), nullable=False, default={}
  65. )
  66. # One-to-many (or many-to-one?) relationships
  67. parent_asset_id = db.Column(
  68. db.Integer, db.ForeignKey("generic_asset.id", ondelete="CASCADE"), nullable=True
  69. )
  70. generic_asset_type_id = db.Column(
  71. db.Integer, db.ForeignKey("generic_asset_type.id"), nullable=False
  72. )
  73. child_assets = db.relationship(
  74. "GenericAsset",
  75. cascade="all",
  76. backref=db.backref("parent_asset", remote_side="GenericAsset.id"),
  77. )
  78. generic_asset_type = db.relationship(
  79. "GenericAssetType",
  80. foreign_keys=[generic_asset_type_id],
  81. backref=db.backref("generic_assets", lazy=True),
  82. )
  83. # Many-to-many relationships
  84. annotations = db.relationship(
  85. "Annotation",
  86. secondary="annotations_assets",
  87. backref=db.backref("assets", lazy="dynamic"),
  88. )
  89. def __acl__(self):
  90. """
  91. All logged-in users can read if the asset is public.
  92. For non-public assets, we allow reading to whoever can read the account,
  93. and editing for every user in the account.
  94. Deletion is left to account admins.
  95. """
  96. return {
  97. "create-children": f"account:{self.account_id}",
  98. "read": (
  99. self.owner.__acl__()["read"]
  100. if self.account_id is not None
  101. else EVERY_LOGGED_IN_USER
  102. ),
  103. "update": f"account:{self.account_id}",
  104. "delete": (f"account:{self.account_id}", f"role:{ACCOUNT_ADMIN_ROLE}"),
  105. }
  106. def __repr__(self):
  107. return "<GenericAsset %s: %r (%s)>" % (
  108. self.id,
  109. self.name,
  110. self.generic_asset_type.name,
  111. )
  112. def validate_sensors_to_show(
  113. self,
  114. suggest_default_sensors: bool = True,
  115. ) -> list[dict[str, str | None | "Sensor" | list["Sensor"]]]: # noqa: F821
  116. """
  117. Validate and transform the 'sensors_to_show' attribute into the latest format for use in graph-making code.
  118. This function ensures that the 'sensors_to_show' attribute:
  119. 1. Follows the latest format, even if the data in the database uses an older format.
  120. 2. Contains only sensors that the user has access to (based on the current asset, account, or public availability).
  121. Steps:
  122. - The function deserializes the 'sensors_to_show' data from the database, ensuring that older formats are parsed correctly.
  123. - It checks if each sensor is accessible by the user and filters out any unauthorized sensors.
  124. - The sensor structure is rebuilt according to the latest format, which allows for grouping sensors and adding optional titles.
  125. Details on format:
  126. - The 'sensors_to_show' attribute is defined as a list of sensor IDs or nested lists of sensor IDs (to indicate grouping).
  127. - Titles may be associated with rows of sensors. If no title is provided, `{"title": None}` will be assigned.
  128. - Nested lists of sensors indicate that they should be shown together (e.g., layered in the same chart).
  129. Example inputs:
  130. 1. Simple list of sensors, where sensors 42 and 44 are grouped:
  131. sensors_to_show = [40, 35, 41, [42, 44], 43, 45]
  132. 2. List with titles and sensor groupings:
  133. sensors_to_show = [
  134. {"title": "Title 1", "sensor": 40},
  135. {"title": "Title 2", "sensors": [41, 42]},
  136. [43, 44], 45, 46
  137. ]
  138. Parameters:
  139. - suggest_default_sensors: If True, the function will suggest default sensors if 'sensors_to_show' is not set.
  140. - If False, the function will return an empty list if 'sensors_to_show' is not set.
  141. Returned structure:
  142. - The function returns a list of dictionaries, with each dictionary containing either a 'sensor' (for individual sensors) or 'sensors' (for groups of sensors), and an optional 'title'.
  143. - Example output:
  144. [
  145. {"title": "Title 1", "sensor": <Sensor object for sensor 40>},
  146. {"title": "Title 2", "sensors": [<Sensor object for sensor 41>, <Sensor object for sensor 42>]},
  147. {"title": None, "sensors": [<Sensor object for sensor 43>, <Sensor object for sensor 44>]},
  148. {"title": None, "sensor": <Sensor object for sensor 45>},
  149. {"title": None, "sensor": <Sensor object for sensor 46>}
  150. ]
  151. If the 'sensors_to_show' attribute is missing, the function defaults to showing two of the asset's sensors, grouped together if they share the same unit, or separately if not.
  152. If the suggest_default_sensors flag is set to False, the function will not suggest default sensors and will return an empty list if 'sensors_to_show' is not set.
  153. Note:
  154. Unauthorized sensors are filtered out, and a warning is logged. Only sensors the user has permission to access are included in the final result.
  155. """
  156. # If not set, use defaults (show first 2 sensors)
  157. if not self.sensors_to_show and suggest_default_sensors:
  158. sensors_to_show = self.sensors[:2]
  159. if (
  160. len(sensors_to_show) == 2
  161. and sensors_to_show[0].unit == sensors_to_show[1].unit
  162. ):
  163. # Sensors are shown together (e.g. they can share the same y-axis)
  164. return [{"title": None, "sensors": sensors_to_show}]
  165. # Otherwise, show separately
  166. return [{"title": None, "sensors": [sensor]} for sensor in sensors_to_show]
  167. sensor_ids_to_show = self.sensors_to_show
  168. # Import the schema for validation
  169. from flexmeasures.data.schemas.generic_assets import SensorsToShowSchema
  170. sensors_to_show_schema = SensorsToShowSchema()
  171. # Deserialize the sensor_ids_to_show using SensorsToShowSchema
  172. standardized_sensors_to_show = sensors_to_show_schema.deserialize(
  173. sensor_ids_to_show
  174. )
  175. sensor_id_allowlist = SensorsToShowSchema.flatten(standardized_sensors_to_show)
  176. # Only allow showing sensors from assets owned by the user's organization,
  177. # except in play mode, where any sensor may be shown
  178. accounts = [self.owner] if self.owner is not None else None
  179. if current_app.config.get("FLEXMEASURES_MODE") == "play":
  180. from flexmeasures.data.models.user import Account
  181. accounts = db.session.scalars(select(Account)).all()
  182. from flexmeasures.data.services.sensors import get_sensors
  183. accessible_sensor_map = {
  184. sensor.id: sensor
  185. for sensor in get_sensors(
  186. account=accounts,
  187. include_public_assets=True,
  188. sensor_id_allowlist=sensor_id_allowlist,
  189. )
  190. }
  191. # Build list of sensor objects that are accessible
  192. sensors_to_show = []
  193. missed_sensor_ids = []
  194. for entry in standardized_sensors_to_show:
  195. title = entry.get("title")
  196. sensors = entry.get("sensors")
  197. accessible_sensors = [
  198. accessible_sensor_map.get(sid)
  199. for sid in sensors
  200. if sid in accessible_sensor_map
  201. ]
  202. inaccessible = [sid for sid in sensors if sid not in accessible_sensor_map]
  203. missed_sensor_ids.extend(inaccessible)
  204. if accessible_sensors:
  205. sensors_to_show.append({"title": title, "sensors": accessible_sensors})
  206. if missed_sensor_ids:
  207. current_app.logger.warning(
  208. f"Cannot include sensor(s) {missed_sensor_ids} in sensors_to_show on asset {self}, as it is not accessible to user {current_user}."
  209. )
  210. return sensors_to_show
  211. @property
  212. def asset_type(self) -> GenericAssetType:
  213. """This property prepares for dropping the "generic" prefix later"""
  214. return self.generic_asset_type
  215. account_id = db.Column(
  216. db.Integer, db.ForeignKey("account.id", ondelete="CASCADE"), nullable=True
  217. ) # if null, asset is public
  218. owner = db.relationship(
  219. "Account",
  220. backref=db.backref(
  221. "generic_assets",
  222. foreign_keys=[account_id],
  223. lazy=True,
  224. cascade="all, delete-orphan",
  225. passive_deletes=True,
  226. ),
  227. )
  228. def get_path(self, separator: str = ">") -> str:
  229. if self.parent_asset is not None:
  230. return f"{self.parent_asset.get_path(separator=separator)}{separator}{self.name}"
  231. elif self.owner is None:
  232. return f"PUBLIC{separator}{self.name}"
  233. else:
  234. return f"{self.owner.get_path(separator=separator)}{separator}{self.name}"
  235. @property
  236. def offspring(self) -> list[GenericAsset]:
  237. """Returns a flattened list of all offspring, which is looked up recursively."""
  238. offspring = []
  239. for child in self.child_assets:
  240. offspring.extend(child.offspring)
  241. return offspring + self.child_assets
  242. @property
  243. def location(self) -> tuple[float, float] | None:
  244. location = (self.latitude, self.longitude)
  245. if None not in location:
  246. return location
  247. @hybrid_method
  248. def great_circle_distance(self, **kwargs):
  249. """Query great circle distance (in km).
  250. Can be called with an object that has latitude and longitude properties, for example:
  251. great_circle_distance(object=asset)
  252. Can also be called with latitude and longitude parameters, for example:
  253. great_circle_distance(latitude=32, longitude=54)
  254. great_circle_distance(lat=32, lng=54)
  255. """
  256. other_location = geo_utils.parse_lat_lng(kwargs)
  257. if None in other_location:
  258. return None
  259. return geo_utils.earth_distance(self.location, other_location)
  260. @great_circle_distance.expression
  261. def great_circle_distance(self, **kwargs):
  262. """Query great circle distance (unclear if in km or in miles).
  263. Can be called with an object that has latitude and longitude properties, for example:
  264. great_circle_distance(object=asset)
  265. Can also be called with latitude and longitude parameters, for example:
  266. great_circle_distance(latitude=32, longitude=54)
  267. great_circle_distance(lat=32, lng=54)
  268. Requires the following Postgres extensions: earthdistance and cube.
  269. """
  270. other_location = geo_utils.parse_lat_lng(kwargs)
  271. if None in other_location:
  272. return None
  273. return func.earth_distance(
  274. func.ll_to_earth(self.latitude, self.longitude),
  275. func.ll_to_earth(*other_location),
  276. )
  277. def get_attribute(self, attribute: str, default: Any = None):
  278. if attribute in self.attributes:
  279. return self.attributes[attribute]
  280. if self.flex_context and attribute in self.flex_context:
  281. return self.flex_context[attribute]
  282. return default
  283. def has_attribute(self, attribute: str) -> bool:
  284. return attribute in self.attributes
  285. def set_attribute(self, attribute: str, value):
  286. if self.has_attribute(attribute):
  287. self.attributes[attribute] = value
  288. def get_flex_context(self) -> dict:
  289. """Reconstitutes the asset's serialized flex-context by gathering flex-contexts upwards in the asset tree.
  290. Flex-context fields of ancestors that are nearer have priority.
  291. We return once we collect all flex-context fields or reach the top asset.
  292. """
  293. from flexmeasures.data.schemas.scheduling import DBFlexContextSchema
  294. flex_context_field_names = set(DBFlexContextSchema.mapped_schema_keys.values())
  295. if self.flex_context:
  296. flex_context = self.flex_context.copy()
  297. else:
  298. flex_context = {}
  299. parent_asset = self.parent_asset
  300. while set(flex_context.keys()) != flex_context_field_names and parent_asset:
  301. flex_context = {**parent_asset.flex_context, **flex_context}
  302. parent_asset = parent_asset.parent_asset
  303. return flex_context
  304. def get_consumption_price_sensor(self):
  305. """Searches for consumption_price_sensor upwards on the asset tree"""
  306. from flexmeasures.data.models.time_series import Sensor
  307. flex_context = self.get_flex_context()
  308. sensor_id = flex_context.get("consumption-price-sensor")
  309. if sensor_id is None:
  310. consumption_price_data = flex_context.get("consumption-price")
  311. if consumption_price_data:
  312. sensor_id = consumption_price_data.get("sensor")
  313. if sensor_id:
  314. return Sensor.query.get(sensor_id) or None
  315. if self.parent_asset:
  316. return self.parent_asset.get_consumption_price_sensor()
  317. return None
  318. def get_production_price_sensor(self):
  319. """Searches for production_price_sensor upwards on the asset tree"""
  320. from flexmeasures.data.models.time_series import Sensor
  321. flex_context = self.get_flex_context()
  322. sensor_id = flex_context.get("production-price-sensor")
  323. if sensor_id is None:
  324. production_price_data = flex_context.get("production-price")
  325. if production_price_data:
  326. sensor_id = production_price_data.get("sensor")
  327. if sensor_id:
  328. return Sensor.query.get(sensor_id) or None
  329. if self.parent_asset:
  330. return self.parent_asset.get_production_price_sensor()
  331. return None
  332. def get_inflexible_device_sensors(self):
  333. """
  334. Searches for inflexible_device_sensors upwards on the asset tree
  335. This search will stop once any sensors are found (will not aggregate towards the top of the tree)
  336. """
  337. from flexmeasures.data.models.time_series import Sensor
  338. flex_context = self.get_flex_context()
  339. # Need to load inflexible_device_sensors manually as generic_asset does not get to SQLAlchemy session context.
  340. if flex_context.get("inflexible-device-sensors"):
  341. sensors = Sensor.query.filter(
  342. Sensor.id.in_(flex_context["inflexible-device-sensors"])
  343. ).all()
  344. return sensors or []
  345. if self.parent_asset:
  346. return self.parent_asset.get_inflexible_device_sensors()
  347. return []
  348. @property
  349. def has_power_sensors(self) -> bool:
  350. """True if at least one power sensor is attached"""
  351. return any([s.measures_power for s in self.sensors])
  352. @property
  353. def has_energy_sensors(self) -> bool:
  354. """True if at least one energy sensor is attached"""
  355. return any([s.measures_energy for s in self.sensors])
  356. def add_annotations(
  357. self,
  358. df: pd.DataFrame,
  359. annotation_type: str,
  360. commit_transaction: bool = False,
  361. ):
  362. """Add a data frame describing annotations to the database, and assign the annotations to this asset."""
  363. annotations = Annotation.add(df, annotation_type=annotation_type)
  364. self.annotations += annotations
  365. db.session.add(self)
  366. if commit_transaction:
  367. db.session.commit()
  368. def search_annotations(
  369. self,
  370. annotations_after: datetime | None = None,
  371. annotations_before: datetime | None = None,
  372. source: (
  373. DataSource | list[DataSource] | int | list[int] | str | list[str] | None
  374. ) = None,
  375. annotation_type: str = None,
  376. include_account_annotations: bool = False,
  377. as_frame: bool = False,
  378. ) -> list[Annotation] | pd.DataFrame:
  379. """Return annotations assigned to this asset, and optionally, also those assigned to the asset's account.
  380. The returned annotations do not include any annotations on public accounts.
  381. :param annotations_after: only return annotations that end after this datetime (exclusive)
  382. :param annotations_before: only return annotations that start before this datetime (exclusive)
  383. """
  384. parsed_sources = parse_source_arg(source)
  385. annotations = db.session.scalars(
  386. query_asset_annotations(
  387. asset_id=self.id,
  388. annotations_after=annotations_after,
  389. annotations_before=annotations_before,
  390. sources=parsed_sources,
  391. annotation_type=annotation_type,
  392. )
  393. ).all()
  394. if include_account_annotations and self.owner is not None:
  395. annotations += self.owner.search_annotations(
  396. annotations_after=annotations_after,
  397. annotations_before=annotations_before,
  398. source=source,
  399. )
  400. return to_annotation_frame(annotations) if as_frame else annotations
  401. def count_annotations(
  402. self,
  403. annotation_starts_after: datetime | None = None, # deprecated
  404. annotations_after: datetime | None = None,
  405. annotation_ends_before: datetime | None = None, # deprecated
  406. annotations_before: datetime | None = None,
  407. source: (
  408. DataSource | list[DataSource] | int | list[int] | str | list[str] | None
  409. ) = None,
  410. annotation_type: str = None,
  411. ) -> int:
  412. """Count the number of annotations assigned to this asset."""
  413. # todo: deprecate the 'annotation_starts_after' argument in favor of 'annotations_after' (announced v0.11.0)
  414. annotations_after = tb_utils.replace_deprecated_argument(
  415. "annotation_starts_after",
  416. annotation_starts_after,
  417. "annotations_after",
  418. annotations_after,
  419. required_argument=False,
  420. )
  421. # todo: deprecate the 'annotation_ends_before' argument in favor of 'annotations_before' (announced v0.11.0)
  422. annotations_before = tb_utils.replace_deprecated_argument(
  423. "annotation_ends_before",
  424. annotation_ends_before,
  425. "annotations_before",
  426. annotations_before,
  427. required_argument=False,
  428. )
  429. parsed_sources = parse_source_arg(source)
  430. return query_asset_annotations(
  431. asset_id=self.id,
  432. annotations_after=annotations_after,
  433. annotations_before=annotations_before,
  434. sources=parsed_sources,
  435. annotation_type=annotation_type,
  436. ).count()
  437. def chart(
  438. self,
  439. chart_type: str = "chart_for_multiple_sensors",
  440. event_starts_after: datetime | None = None,
  441. event_ends_before: datetime | None = None,
  442. beliefs_after: datetime | None = None,
  443. beliefs_before: datetime | None = None,
  444. combine_legend: bool = True,
  445. source: (
  446. DataSource | list[DataSource] | int | list[int] | str | list[str] | None
  447. ) = None,
  448. include_data: bool = False,
  449. dataset_name: str | None = None,
  450. resolution: str | timedelta | None = None,
  451. **kwargs,
  452. ) -> dict:
  453. """Create a vega-lite chart showing sensor data.
  454. :param chart_type: currently only "bar_chart" # todo: where can we properly list the available chart types?
  455. :param event_starts_after: only return beliefs about events that start after this datetime (inclusive)
  456. :param event_ends_before: only return beliefs about events that end before this datetime (inclusive)
  457. :param beliefs_after: only return beliefs formed after this datetime (inclusive)
  458. :param beliefs_before: only return beliefs formed before this datetime (inclusive)
  459. :param combine_legend: show a combined legend of all plots below the chart
  460. :param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources
  461. :param include_data: if True, include data in the chart, or if False, exclude data
  462. :param dataset_name: optionally name the dataset used in the chart (the default name is sensor_<id>)
  463. :param resolution: optionally set the resolution of data being displayed
  464. :returns: JSON string defining vega-lite chart specs
  465. """
  466. processed_sensors_to_show = self.validate_sensors_to_show()
  467. sensors = flatten_unique(processed_sensors_to_show)
  468. for sensor in sensors:
  469. sensor.sensor_type = sensor.get_attribute("sensor_type", sensor.name)
  470. # Set up chart specification
  471. if dataset_name is None:
  472. dataset_name = "asset_" + str(self.id)
  473. if event_starts_after:
  474. kwargs["event_starts_after"] = event_starts_after
  475. if event_ends_before:
  476. kwargs["event_ends_before"] = event_ends_before
  477. chart_specs = chart_type_to_chart_specs(
  478. chart_type,
  479. sensors_to_show=processed_sensors_to_show,
  480. dataset_name=dataset_name,
  481. combine_legend=combine_legend,
  482. **kwargs,
  483. )
  484. if include_data:
  485. # Get data
  486. data = self.search_beliefs(
  487. sensors=sensors,
  488. as_json=True,
  489. event_starts_after=event_starts_after,
  490. event_ends_before=event_ends_before,
  491. beliefs_after=beliefs_after,
  492. beliefs_before=beliefs_before,
  493. source=source,
  494. resolution=resolution,
  495. )
  496. # Combine chart specs and data
  497. chart_specs["datasets"] = {
  498. dataset_name: json.loads(data),
  499. }
  500. return chart_specs
  501. def search_beliefs(
  502. self,
  503. sensors: list["Sensor"] | None = None, # noqa F821
  504. event_starts_after: datetime | None = None,
  505. event_ends_before: datetime | None = None,
  506. beliefs_after: datetime | None = None,
  507. beliefs_before: datetime | None = None,
  508. horizons_at_least: timedelta | None = None,
  509. horizons_at_most: timedelta | None = None,
  510. source: (
  511. DataSource | list[DataSource] | int | list[int] | str | list[str] | None
  512. ) = None,
  513. most_recent_beliefs_only: bool = True,
  514. most_recent_events_only: bool = False,
  515. as_json: bool = False,
  516. resolution: timedelta | None = None,
  517. ) -> BeliefsDataFrame | str:
  518. """Search all beliefs about events for all sensors of this asset
  519. If you don't set any filters, you get the most recent beliefs about all events.
  520. :param sensors: only return beliefs about events registered by these sensors
  521. :param event_starts_after: only return beliefs about events that start after this datetime (inclusive)
  522. :param event_ends_before: only return beliefs about events that end before this datetime (inclusive)
  523. :param beliefs_after: only return beliefs formed after this datetime (inclusive)
  524. :param beliefs_before: only return beliefs formed before this datetime (inclusive)
  525. :param horizons_at_least: only return beliefs with a belief horizon equal or greater than this timedelta (for example, use timedelta(0) to get ante knowledge time beliefs)
  526. :param horizons_at_most: only return beliefs with a belief horizon equal or less than this timedelta (for example, use timedelta(0) to get post knowledge time beliefs)
  527. :param source: search only beliefs by this source (pass the DataSource, or its name or id) or list of sources
  528. :param most_recent_events_only: only return (post knowledge time) beliefs for the most recent event (maximum event start)
  529. :param as_json: return beliefs in JSON format (e.g. for use in charts) rather than as BeliefsDataFrame
  530. :param resolution: optionally set the resolution of data being displayed
  531. :returns: dictionary of BeliefsDataFrames or JSON string (if as_json is True)
  532. """
  533. bdf_dict = {}
  534. if sensors is None:
  535. sensors = self.sensors
  536. for sensor in sensors:
  537. bdf_dict[sensor] = sensor.search_beliefs(
  538. event_starts_after=event_starts_after,
  539. event_ends_before=event_ends_before,
  540. beliefs_after=beliefs_after,
  541. beliefs_before=beliefs_before,
  542. horizons_at_least=horizons_at_least,
  543. horizons_at_most=horizons_at_most,
  544. source=source,
  545. most_recent_beliefs_only=most_recent_beliefs_only,
  546. most_recent_events_only=most_recent_events_only,
  547. one_deterministic_belief_per_event_per_source=True,
  548. resolution=resolution,
  549. )
  550. if as_json:
  551. from flexmeasures.data.services.time_series import simplify_index
  552. if sensors:
  553. minimum_resampling_resolution = determine_minimum_resampling_resolution(
  554. [bdf.event_resolution for bdf in bdf_dict.values()]
  555. )
  556. if resolution is not None:
  557. minimum_resampling_resolution = resolution
  558. df_dict = {}
  559. for sensor, bdf in bdf_dict.items():
  560. if bdf.event_resolution > timedelta(0):
  561. bdf = bdf.resample_events(minimum_resampling_resolution)
  562. bdf["belief_horizon"] = bdf.belief_horizons.to_numpy()
  563. df = simplify_index(
  564. bdf,
  565. index_levels_to_columns=(
  566. ["source"]
  567. if most_recent_beliefs_only
  568. else ["belief_time", "source"]
  569. ),
  570. ).set_index(
  571. (
  572. ["source"]
  573. if most_recent_beliefs_only
  574. else ["belief_time", "source"]
  575. ),
  576. append=True,
  577. )
  578. df["sensor"] = sensor # or some JSONifiable representation
  579. df = df.set_index(["sensor"], append=True)
  580. df_dict[sensor.id] = df
  581. df = pd.concat(df_dict.values())
  582. else:
  583. df = simplify_index(
  584. BeliefsDataFrame(),
  585. index_levels_to_columns=(
  586. ["source"]
  587. if most_recent_beliefs_only
  588. else ["belief_time", "source"]
  589. ),
  590. ).set_index(
  591. (
  592. ["source"]
  593. if most_recent_beliefs_only
  594. else ["belief_time", "source"]
  595. ),
  596. append=True,
  597. )
  598. df["sensor"] = {} # ensure the same columns as a non-empty frame
  599. df = df.reset_index()
  600. df["source"] = df["source"].apply(lambda x: x.to_dict())
  601. df["sensor"] = df["sensor"].apply(lambda x: x.to_dict())
  602. return df.to_json(orient="records")
  603. return bdf_dict
  604. @property
  605. def timezone(
  606. self,
  607. ) -> str:
  608. """Timezone relevant to the asset.
  609. If a timezone is not given as an attribute of the asset, it is taken from one of its sensors.
  610. """
  611. if self.has_attribute("timezone"):
  612. return self.get_attribute("timezone")
  613. if self.sensors:
  614. return self.sensors[0].timezone
  615. return "UTC"
  616. @property
  617. def timerange(self) -> dict[str, datetime]:
  618. """Time range for which sensor data exists.
  619. :returns: dictionary with start and end, for example:
  620. {
  621. 'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc),
  622. 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc)
  623. }
  624. """
  625. return self.get_timerange(self.sensors)
  626. @property
  627. def timerange_of_sensors_to_show(self) -> dict[str, datetime]:
  628. """Time range for which sensor data exists, for sensors to show.
  629. :returns: dictionary with start and end, for example:
  630. {
  631. 'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc),
  632. 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc)
  633. }
  634. """
  635. return self.get_timerange(self.validate_sensors_to_show())
  636. @classmethod
  637. def get_timerange(cls, sensors: list["Sensor"]) -> dict[str, datetime]: # noqa F821
  638. """Time range for which sensor data exists.
  639. :param sensors: sensors to check
  640. :returns: dictionary with start and end, for example:
  641. {
  642. 'start': datetime.datetime(2020, 12, 3, 14, 0, tzinfo=pytz.utc),
  643. 'end': datetime.datetime(2020, 12, 3, 14, 30, tzinfo=pytz.utc)
  644. }
  645. """
  646. sensor_ids = [s.id for s in flatten_unique(sensors)]
  647. start, end = get_timerange(sensor_ids)
  648. return dict(start=start, end=end)
  649. def set_inflexible_sensors(self, inflexible_sensor_ids: list[int]) -> None:
  650. """Set inflexible sensors for this asset.
  651. :param inflexible_sensor_ids: list of sensor ids
  652. """
  653. from flexmeasures.data.models.time_series import Sensor
  654. # -1 choice corresponds to "--Select sensor id--" which means no sensor is selected
  655. # and all linked sensors should be unlinked
  656. if len(inflexible_sensor_ids) == 1 and inflexible_sensor_ids[0] == -1:
  657. self.inflexible_device_sensors = []
  658. else:
  659. self.inflexible_device_sensors = Sensor.query.filter(
  660. Sensor.id.in_(inflexible_sensor_ids)
  661. ).all()
  662. db.session.add(self)
  663. def create_generic_asset(generic_asset_type: str, **kwargs) -> GenericAsset:
  664. """Create a GenericAsset and assigns it an id.
  665. :param generic_asset_type: "asset", "market" or "weather_sensor"
  666. :param kwargs: should have values for keys "name", and:
  667. - "asset_type_name" or "asset_type" when generic_asset_type is "asset"
  668. - "market_type_name" or "market_type" when generic_asset_type is "market"
  669. - "weather_sensor_type_name" or "weather_sensor_type" when generic_asset_type is "weather_sensor"
  670. - alternatively, "sensor_type" is also fine
  671. :returns: the created GenericAsset
  672. """
  673. asset_type_name = kwargs.pop(f"{generic_asset_type}_type_name", None)
  674. if asset_type_name is None:
  675. if f"{generic_asset_type}_type" in kwargs:
  676. asset_type_name = kwargs.pop(f"{generic_asset_type}_type").name
  677. else:
  678. asset_type_name = kwargs.pop("sensor_type").name
  679. generic_asset_type = db.session.execute(
  680. select(GenericAssetType).filter_by(name=asset_type_name)
  681. ).scalar_one_or_none()
  682. if generic_asset_type is None:
  683. raise ValueError(f"Cannot find GenericAssetType {asset_type_name} in database.")
  684. new_generic_asset = GenericAsset(
  685. name=kwargs["name"],
  686. generic_asset_type_id=generic_asset_type.id,
  687. attributes=kwargs["attributes"] if "attributes" in kwargs else {},
  688. )
  689. for arg in ("latitude", "longitude", "account_id"):
  690. if arg in kwargs:
  691. setattr(new_generic_asset, arg, kwargs[arg])
  692. db.session.add(new_generic_asset)
  693. db.session.flush() # generates the pkey for new_generic_asset
  694. return new_generic_asset
  695. def assets_share_location(assets: list[GenericAsset]) -> bool:
  696. """
  697. Return True if all assets in this list are located on the same spot.
  698. TODO: In the future, we might soften this to compare if assets are in the same "housing" or "site".
  699. """
  700. if not assets:
  701. return True
  702. return all([a.location == assets[0].location for a in assets])
  703. def get_center_location_of_assets(user: User | None) -> tuple[float, float]:
  704. """
  705. Find the center position between all generic assets of the user's account.
  706. """
  707. query = (
  708. "Select (min(latitude) + max(latitude)) / 2 as latitude,"
  709. " (min(longitude) + max(longitude)) / 2 as longitude"
  710. " from generic_asset"
  711. )
  712. if user is None:
  713. user = current_user
  714. query += f" where generic_asset.account_id = {user.account_id}"
  715. locations: list[Row] = db.session.execute(text(query + ";")).fetchall()
  716. if (
  717. len(locations) == 0
  718. or locations[0].latitude is None
  719. or locations[0].longitude is None
  720. ):
  721. return 52.366, 4.904 # Amsterdam, NL
  722. return locations[0].latitude, locations[0].longitude