SQL query to get latest non null value of columns grouping by a column

Question

I have a table in MySQL table, which looks like

+--------------------------+------------+-----------------+---------+------+-------+-------------+-------------------------+----------------------------+-------+-------+
| deviceID                 | date       | timestamp       | counter | rssi | vavId | nvo_airflow | nvo_air_damper_position | nvo_temperature_sensor_pps | block | floor |
+--------------------------+------------+-----------------+---------+------+-------+-------------+-------------------------+----------------------------+-------+-------+
| fd00::212:4b00:1957:d616 | 2020-02-29 | 12:40:01.513066 |     805 |   91 |     7 |        NULL |                    NULL |                      26.49 | NULL  | ABCD  |
| fd00::212:4b00:1957:d616 | 2020-02-29 | 12:41:01.542272 |     807 |   94 |     5 |        50   |                    64   |                      26.37 | NULL  | ABCD  |
| fd00::212:4b00:1957:d616 | 2020-02-29 | 12:43:01.699023 |     811 |   90 |     7 |        50   |                    NULL |                       NULL | NULL  | ABCD  |
| fd00::212:4b00:1957:d616 | 2020-02-29 | 12:46:01.412259 |     817 |   64 |    26 |        NULL |                    NULL |                      25.85 | NULL  | ABCD  |
| fd00::212:4b00:1957:d616 | 2020-02-29 | 12:48:01.576133 |     821 |   91 |    26 |        55   |                     42  |                       NULL | NULL  | ABCD  |
| fd00::212:4b00:1957:d616 | 2020-02-29 | 12:49:01.529593 |     823 |   91 |     7 |        45   |                     72  |                       NULL | NULL  | ABCD  |

I want to get the latest non null data of 3 columns(nvo_airflow, nvo_air_damper_position, nvo_temperature_sensor_pps) for each vavId.

My result should look something like

vavId,nvo_airflow,nvo_air_damper_position,nvo_temperature_sensor_pps
 5,50,64,26.37
 7,45,72,26.49
26,55,42,25.85

I have written a sql query for the same,

SELECT airflow_table.nvo_airflow,damper_position_table.nvo_air_damper_position,temperature_sensor_table.nvo_temperature_sensor_pps,temperature_sensor_table.vavId
FROM(
    ((SELECT t1.date,t1.timestamp,t1.nvo_airflow,t1.vavId
    FROM
        (SELECT * FROM vavDataOptimized where date='2020-02-29')t1
    INNER JOIN
        (SELECT max(timestamp) as recent_timestamp,vavId FROM vavDataOptimized where date='2020-02-29' and `nvo_airflow` is not null GROUP BY vavId)t2
        ON (t1.timestamp = t2.recent_timestamp and t1.vavId = t2.vavId)
    ORDER BY vavId) airflow_table
inner join 
    (SELECT t1.date,t1.timestamp,t1.nvo_air_damper_position,t1.vavId
    FROM
        (SELECT * FROM vavDataOptimized where date='2020-02-29')t1
    INNER JOIN
        (SELECT max(timestamp) as recent_timestamp,vavId FROM vavDataOptimized where date='2020-02-29' and `nvo_air_damper_position` is not null GROUP BY vavId)t2
        ON (t1.timestamp = t2.recent_timestamp and t1.vavId = t2.vavId)
    ORDER BY vavId) damper_position_table ON airflow_table.vavId = damper_position_table.vavId)
inner join
    (SELECT t1.date,t1.timestamp,t1.nvo_temperature_sensor_pps,t1.vavId
    FROM
        (SELECT * FROM vavDataOptimized where date='2020-02-29')t1
    INNER JOIN
        (SELECT max(timestamp) as recent_timestamp,vavId FROM vavDataOptimized where date='2020-02-29' and `nvo_temperature_sensor_pps` is not null GROUP BY vavId)t2
        ON (t1.timestamp = t2.recent_timestamp and t1.vavId = t2.vavId)
    ORDER BY vavId) temperature_sensor_table on airflow_table.vavId = temperature_sensor_table.vavId);

What I am trying to do is getting the latest value for each of nvo_airflow, nvo_air_damper_position, nvo_temperature_sensor_pps for each vav as three intermediate tables and then trying to do a inner join on the tables.

This query is taking a lot of time to time and not getting executed. I am not sure if I am doing in it an optimized way. Am I doing something wrong, or is there a better way of doing this?

Answer 1

Does something like this fits your needs?

select vavId,nvo_airflow,nvo_air_damper_position,nvo_temperature_sensor_pps
from
    (select vavId, @rownum1 := @rownum1 + 1 as rownum1 from
        (select vavId 
        from vavDataOptimized 
        where vavId is not NULL
        ORDER BY date1,timestamp1 DESC LIMIT 3) a
            CROSS JOIN (SELECT @rownum1 := 0) v) a,
    (select nvo_airflow, @rownum2 := @rownum2 + 1 as rownum2 from
        (select nvo_airflow 
        from vavDataOptimized 
        where nvo_airflow is not NULL
        ORDER BY date1,timestamp1 DESC LIMIT 3) b
            CROSS JOIN (SELECT @rownum2 := 0) v) b,
    (select nvo_air_damper_position, @rownum3 := @rownum3 + 1 as rownum3 from
        (select nvo_air_damper_position 
        from vavDataOptimized 
        where nvo_air_damper_position is not NULL
        ORDER BY date1,timestamp1 DESC LIMIT 3) c
            CROSS JOIN (SELECT @rownum3 := 0) v) c,
    (select nvo_temperature_sensor_pps, @rownum4 := @rownum4 + 1 as rownum4 from
        (select nvo_temperature_sensor_pps 
        from vavDataOptimized 
        where nvo_temperature_sensor_pps is not NULL
        ORDER BY date1,timestamp1 DESC LIMIT 3) d
            CROSS JOIN (SELECT @rownum4 := 0) v) d
where rownum1 = rownum2
and rownum1 = rownum3
and rownum1 = rownum4

Here is the fiddle : https://dbfiddle.uk/?rdbms=mysql_8.0&fiddle=57539e363b668038547df037b15f0dee

Answer 2

Here is one option of doing it. What i have done is to first rank the records on the basis of latest value of the attribute columns which are not null(eg: airflow_flg=1 implies ranks by not null values only)

After that a union of all the three would get the values you are looking for.

with data
  as (
select *
      ,case when nvo_airflow is null then 0 else 1 end as airflow_flg
      ,case when nvo_air_damper_position is null then 0 else 1 end as damper_flg
      ,case when nvo_temperature_sensor_pps is null then 0 else 1 end as sensor_flg
      ,row_number() over(partition by case when nvo_airflow is null then 0 else 1 end,deviceid,vavid order by timestamp1 desc) as rnk_airflow
      ,row_number() over(partition by case when nvo_air_damper_position is null then 0 else 1 end,deviceid,vavid order by timestamp1 desc) as rnk_damper
      ,row_number() over(partition by case when nvo_temperature_sensor_pps is null then 0 else 1 end,deviceid,vavid order by timestamp1 desc) as rnk_sensor
  from t
     )
  ,concat_data
   as (
 select deviceid
        ,vavid
        ,nvo_airflow as val
        ,'nvo_airflow' as txt
   from data
 where airflow_flg=1   
   and rnk_airflow=1
 union all   
 select deviceid
        ,vavid
        ,nvo_air_damper_position as val
        ,'nvo_air_damper_position' as txt
   from data
 where damper_flg=1   
   and rnk_damper=1
 union all   
 select deviceid
        ,vavid
        ,nvo_temperature_sensor_pps as val
        ,'nvo_temperature_sensor_pps' as txt
   from data
 where sensor_flg=1   
   and rnk_sensor=1
       )
  select deviceid
        ,vavid
        ,max(case when txt='nvo_airflow' then val end) as nvo_airflow
        ,max(case when txt='nvo_air_damper_position' then val end) as nvo_air_damper_position
        ,max(case when txt='nvo_temperature_sensor_pps' then val end) as nvo_temperature_sensor_pps
  from concat_data
  group by deviceid
          ,vavid

https://dbfiddle.uk/?rdbms=mysql_8.0&fiddle=5440a6e55e7b9a82596f57840dc38083