Skip to content

Commit

Permalink
aggregation hourly
Browse files Browse the repository at this point in the history
  • Loading branch information
@ftomei
ftomei committed Sep 23, 2024
1 parent b3f2109 commit 48145c4
Show file tree
Hide file tree
Showing 4 changed files with 125 additions and 42 deletions.
130 changes: 102 additions & 28 deletions agrolib/climate/climate.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#include <QString>
#include <QDate>
#include <QtSql>
#include <qDebug>

#include <math.h> /* ceil */

#include "commonConstants.h"
Expand All @@ -11,7 +13,7 @@
#include "statistics.h"
#include "quality.h"
#include "dbClimate.h"
#include "qdebug.h"
#include "meteo.h"

using namespace std;

Expand Down Expand Up @@ -828,6 +830,69 @@ float loadDailyVarSeries(QString *myError, Crit3DMeteoPointsDbHandler *meteoPoin
}


float loadHourlyVarSeries_SaveOutput(QString *myError, Crit3DMeteoPointsDbHandler *meteoPointsDbHandler,
Crit3DMeteoGridDbHandler *meteoGridDbHandler, Crit3DMeteoPoint* meteoPoint, bool isMeteoGrid,
meteoVariable variable, QDateTime first, QDateTime last, std::vector<float> &outputValues)
{
std::vector<float> hourlyValues;
QDateTime firstDateTimeDB;

if (isMeteoGrid)
{
if (meteoGridDbHandler->gridStructure().isFixedFields())
{
hourlyValues = meteoGridDbHandler->loadGridHourlyVarFixedFields(myError, QString::fromStdString(meteoPoint->id),
variable, first, last, &firstDateTimeDB);
}
else
{
hourlyValues = meteoGridDbHandler->loadGridHourlyVar(myError, QString::fromStdString(meteoPoint->id), variable,
first, last, &firstDateTimeDB);
}
}
else
{
// meteoPoints
hourlyValues = meteoPointsDbHandler->loadHourlyVar(myError, variable, getCrit3DDate(first.date()), getCrit3DDate(last.date()),
&firstDateTimeDB, meteoPoint);
}

// no values
if (hourlyValues.empty())
return 0;

int nrOfDays = first.daysTo(last) + 1;
meteoPoint->initializeObsDataH(meteoPoint->hourlyFraction, nrOfDays, getCrit3DDate(firstDateTimeDB.date()));
meteoPoint->initializeObsDataD(nrOfDays, getCrit3DDate(firstDateTimeDB.date()));

int nrRequestedValues = nrOfDays * 24;
int nrValidValues = 0;
Crit3DQuality qualityCheck;

Crit3DTime currentDateTime = getCrit3DTime(firstDateTimeDB);
for (unsigned int i = 0; i < hourlyValues.size(); i++)
{
quality::qualityType myQuality = qualityCheck.syntacticQualitySingleValue(variable, hourlyValues[i]);
if (myQuality == quality::accepted)
{
nrValidValues = nrValidValues + 1;
meteoPoint->setMeteoPointValueH(currentDateTime.date, currentDateTime.getHour(), currentDateTime.getMinutes(), variable, hourlyValues[i]);
outputValues.push_back(hourlyValues[i]);
}
else
{
meteoPoint->setMeteoPointValueH(currentDateTime.date, currentDateTime.getHour(), currentDateTime.getMinutes(), variable, NODATA);
outputValues.push_back(NODATA);
}

currentDateTime = currentDateTime.addSeconds(3600);
}

float percValue = float(nrValidValues) / float(nrRequestedValues);
return percValue;
}


float loadDailyVarSeries_SaveOutput(QString *myError, Crit3DMeteoPointsDbHandler *meteoPointsDbHandler,
Crit3DMeteoGridDbHandler *meteoGridDbHandler, Crit3DMeteoPoint* meteoPoint, bool isMeteoGrid,
meteoVariable variable, QDate first, QDate last, std::vector<float> &outputValues)
Expand Down Expand Up @@ -857,7 +922,6 @@ float loadDailyVarSeries_SaveOutput(QString *myError, Crit3DMeteoPointsDbHandler
dailyValues = meteoPointsDbHandler->loadDailyVar(variable, getCrit3DDate(first), getCrit3DDate(last), *meteoPoint, firstDateDB);
}


if ( dailyValues.empty() )
{
return 0;
Expand Down Expand Up @@ -893,6 +957,7 @@ float loadDailyVarSeries_SaveOutput(QString *myError, Crit3DMeteoPointsDbHandler
}
}


float loadFromMp_SaveOutput(Crit3DMeteoPoint* meteoPoint,
meteoVariable variable, QDate first, QDate last, std::vector<float> &outputValues)
{
Expand Down Expand Up @@ -955,7 +1020,7 @@ float loadHourlyVarSeries(QString *myError, Crit3DMeteoPointsDbHandler* meteoPoi
if (meteoPoint->nrObsDataDaysH == 0)
{
int nrOfDays = ceil(float(hourlyValues.size()) / float(24 * meteoPoint->hourlyFraction));
meteoPoint->initializeObsDataH(meteoPoint->hourlyFraction, nrOfDays,getCrit3DDate(firstDateDB.date()));
meteoPoint->initializeObsDataH(meteoPoint->hourlyFraction, nrOfDays, getCrit3DDate(firstDateDB.date()));
meteoPoint->initializeObsDataD(nrOfDays, getCrit3DDate(firstDateDB.date()));
}

Expand Down Expand Up @@ -2069,6 +2134,7 @@ std::vector<float> aggregatedHourlyToDailyList(meteoVariable myVar, Crit3DMeteoP

}


bool preElaboration(QString *myError, Crit3DMeteoPointsDbHandler* meteoPointsDbHandler, Crit3DMeteoGridDbHandler* meteoGridDbHandler, Crit3DMeteoPoint* meteoPoint, bool isMeteoGrid, meteoVariable variable, meteoComputation elab1,
QDate startDate, QDate endDate, std::vector<float> &outputValues, float* percValue, Crit3DMeteoSettings* meteoSettings)
{
Expand All @@ -2078,7 +2144,6 @@ bool preElaboration(QString *myError, Crit3DMeteoPointsDbHandler* meteoPointsDbH

switch(variable)
{

case dailyLeafWetness:
{
if ( loadHourlyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, leafWetness, QDateTime(startDate,QTime(1,0,0),Qt::UTC), QDateTime(endDate.addDays(1),QTime(0,0,0),Qt::UTC)) > 0)
Expand Down Expand Up @@ -2120,7 +2185,6 @@ bool preElaboration(QString *myError, Crit3DMeteoPointsDbHandler* meteoPointsDbH
}
case dailyThomAvg: case dailyThomMax: case dailyThomHoursAbove:
{

if (loadHourlyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, airTemperature, QDateTime(startDate,QTime(1,0,0),Qt::UTC), QDateTime(endDate.addDays(1),QTime(0,0,0),Qt::UTC)) > 0)
{
if (loadHourlyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, airRelHumidity, QDateTime(startDate,QTime(1,0,0),Qt::UTC), QDateTime(endDate.addDays(1),QTime(0,0,0),Qt::UTC)) > 0)
Expand All @@ -2132,7 +2196,6 @@ bool preElaboration(QString *myError, Crit3DMeteoPointsDbHandler* meteoPointsDbH
}
case dailyBIC:
{

if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyReferenceEvapotranspirationHS, startDate, endDate) > 0)
{
preElaboration = true;
Expand Down Expand Up @@ -2286,40 +2349,50 @@ bool preElaboration(QString *myError, Crit3DMeteoPointsDbHandler* meteoPointsDbH
break;
}

case winkler: case correctedDegreeDaysSum: case fregoni:
{
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMin, startDate, endDate) > 0 )
case winkler: case correctedDegreeDaysSum: case fregoni:
{
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMax, startDate, endDate) > 0 )
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMin, startDate, endDate) > 0 )
{
preElaboration = true;
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMax, startDate, endDate) > 0 )
{
preElaboration = true;
}
}
break;
}
break;
}

case phenology:
{
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMin, startDate, endDate) > 0 )
case phenology:
{
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMax, startDate, endDate) > 0 )
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMin, startDate, endDate) > 0 )
{
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyPrecipitation, startDate, endDate) > 0 )
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyAirTemperatureMax, startDate, endDate) > 0 )
{
preElaboration = true;
if (loadDailyVarSeries(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, dailyPrecipitation, startDate, endDate) > 0 )
{
preElaboration = true;
}
}
}
}
break;
}
break;
}

default:
{
*percValue = loadDailyVarSeries_SaveOutput(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, variable, startDate, endDate, outputValues);
default:
{
// default variables / elaboration
if (getVarFrequency(variable) == hourly)
{
QDateTime firstTime = QDateTime(startDate, QTime(0, 0, 0, 0));
QDateTime lastTime = QDateTime(startDate, QTime(23, 0, 0, 0));
*percValue = loadHourlyVarSeries_SaveOutput(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, variable, firstTime, lastTime, outputValues);
}
else
{
*percValue = loadDailyVarSeries_SaveOutput(myError, meteoPointsDbHandler, meteoGridDbHandler, meteoPoint, isMeteoGrid, variable, startDate, endDate, outputValues);
}

preElaboration = ((*percValue) > 0);
break;
}
preElaboration = ((*percValue) > 0);
break;
}

}
break;
Expand All @@ -2329,6 +2402,7 @@ bool preElaboration(QString *myError, Crit3DMeteoPointsDbHandler* meteoPointsDbH
return preElaboration;
}


bool preElaborationWithoutLoad(Crit3DMeteoPoint* meteoPoint, meteoVariable variable, QDate startDate, QDate endDate, std::vector<float> &outputValues, float* percValue, Crit3DMeteoSettings* meteoSettings)
{

Expand Down
4 changes: 4 additions & 0 deletions agrolib/climate/climate.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,10 @@
Crit3DMeteoGridDbHandler *meteoGridDbHandler, Crit3DMeteoPoint &meteoPoint, bool isMeteoGrid,
meteoVariable variable, QDate first, QDate last, std::vector<float> &outputValues);

float loadHourlyVarSeries_SaveOutput(QString *myError, Crit3DMeteoPointsDbHandler *meteoPointsDbHandler,
Crit3DMeteoGridDbHandler *meteoGridDbHandler, Crit3DMeteoPoint* meteoPoint, bool isMeteoGrid,
meteoVariable variable, QDateTime first, QDateTime last, std::vector<float> &outputValues);

float loadHourlyVarSeries(QString *myError, Crit3DMeteoPointsDbHandler *meteoPointsDbHandler,
Crit3DMeteoGridDbHandler *meteoGridDbHandler, Crit3DMeteoPoint* meteoPoint,
bool isMeteoGrid, meteoVariable variable, QDateTime first, QDateTime last);
Expand Down
6 changes: 4 additions & 2 deletions agrolib/dbMeteoGrid/dbMeteoGrid.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2700,7 +2700,8 @@ std::vector<float> Crit3DMeteoGridDbHandler::loadGridDailyVarFixedFields(QString
}


std::vector<float> Crit3DMeteoGridDbHandler::loadGridHourlyVar(QString *errorStr, QString meteoPoint, meteoVariable variable, QDateTime first, QDateTime last, QDateTime* firstDateDB)
std::vector<float> Crit3DMeteoGridDbHandler::loadGridHourlyVar(QString *errorStr, QString meteoPoint, meteoVariable variable,
QDateTime first, QDateTime last, QDateTime* firstDateDB)
{

QSqlQuery qry(_db);
Expand Down Expand Up @@ -2787,7 +2788,8 @@ std::vector<float> Crit3DMeteoGridDbHandler::loadGridHourlyVar(QString *errorStr
}


std::vector<float> Crit3DMeteoGridDbHandler::loadGridHourlyVarFixedFields(QString *errorStr, QString meteoPoint, meteoVariable variable, QDateTime first, QDateTime last, QDateTime* firstDateDB)
std::vector<float> Crit3DMeteoGridDbHandler::loadGridHourlyVarFixedFields(QString *errorStr, QString meteoPoint, meteoVariable variable,
QDateTime first, QDateTime last, QDateTime* firstDateDB)
{
QSqlQuery qry(_db);
QString tableH = _tableHourly.prefix + meteoPoint + _tableHourly.postFix;
Expand Down
27 changes: 15 additions & 12 deletions agrolib/dbMeteoPoints/dbMeteoPointsHandler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -775,15 +775,18 @@ std::vector<float> Crit3DMeteoPointsDbHandler::exportAllDataVar(QString *myError
return allDataVarList;
}

std::vector<float> Crit3DMeteoPointsDbHandler::loadHourlyVar(QString *myError, meteoVariable variable, Crit3DDate dateStart, Crit3DDate dateEnd, QDateTime* firstDateDB, Crit3DMeteoPoint *meteoPoint)

std::vector<float> Crit3DMeteoPointsDbHandler::loadHourlyVar(QString *myError, meteoVariable variable,
Crit3DDate dateStart, Crit3DDate dateEnd,
QDateTime* firstDateDB, Crit3DMeteoPoint *meteoPoint)
{
QString dateStr;
QDateTime previousDate;
QDate myDate;
QTime myTime;
float value;
std::vector<float> hourlyVarList;
bool firstRow = true;
bool isFirstRow = true;

int idVar = getIdfromMeteoVar(variable);
QString startDate = QString::fromStdString(dateStart.toISOString());
Expand All @@ -795,7 +798,7 @@ std::vector<float> Crit3DMeteoPointsDbHandler::loadHourlyVar(QString *myError, m

QString statement = QString( "SELECT * FROM `%1` WHERE `%2` = %3 AND date_time >= DATETIME('%4 01:00:00') AND date_time <= DATETIME('%5 00:00:00', '+1 day')")
.arg(tableName, FIELD_METEO_VARIABLE).arg(idVar).arg(startDate, endDate);
if( !qry.exec(statement) )
if(! qry.exec(statement))
{
*myError = qry.lastError().text();
return hourlyVarList;
Expand All @@ -804,37 +807,37 @@ std::vector<float> Crit3DMeteoPointsDbHandler::loadHourlyVar(QString *myError, m
{
while (qry.next())
{
if (firstRow)
if (isFirstRow)
{
dateStr = qry.value(0).toString();
myDate = QDate::fromString(dateStr.mid(0,10), "yyyy-MM-dd");
myTime = QTime::fromString(dateStr.mid(11,8), "HH:mm:ss");
QDateTime d(QDateTime(myDate, myTime, Qt::UTC));
QDateTime currentDate(QDateTime(myDate, myTime, Qt::UTC));

*firstDateDB = d;
previousDate = *firstDateDB;
*firstDateDB = currentDate;
previousDate = currentDate;

value = qry.value(2).toFloat();

hourlyVarList.push_back(value);
firstRow = false;
isFirstRow = false;
}
else
{
dateStr = qry.value(0).toString();
myDate = QDate::fromString(dateStr.mid(0,10), "yyyy-MM-dd");
myTime = QTime::fromString(dateStr.mid(11,8), "HH:mm:ss");
QDateTime d(QDateTime(myDate, myTime, Qt::UTC));
QDateTime currentDate(QDateTime(myDate, myTime, Qt::UTC));

int missingDate = previousDate.daysTo(d);
for (int i =1; i<missingDate; i++)
int missingHours = (currentDate.currentSecsSinceEpoch() - previousDate.currentSecsSinceEpoch()) / 3600;
for (int i=1; i < missingHours; i++)
{
hourlyVarList.push_back(NODATA);
}
value = qry.value(2).toFloat();

hourlyVarList.push_back(value);
previousDate = d;
previousDate = currentDate;
}
}
}
Expand Down

0 comments on commit 48145c4

Please sign in to comment.