// Copyright (C) Stichting Deltares 2024. All rights reserved.
//
// This file is part of the application DAM - UI.
//
// DAM - UI is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
//
// All names, logos, and references to "Deltares" are registered trademarks of
// Stichting Deltares and remain full property of Stichting Deltares at all times.
// All rights reserved.
#if DEBUG
#define AUTO_DELETE_EXISTING_OUTPUT_FOLDER
#endif
using System;
using System.Collections.Generic;
using System.Data;
using System.Data.OleDb;
using System.IO;
using System.Linq;
using CommandLine;
using Deltares.Dam.Data;
using Deltares.Dam.Data.Sensors;
using Deltares.Dam.Tools.CsvToDamxConverter.Properties;
using Deltares.Standard.Application;
using Deltares.Standard.Extensions;
using Deltares.Standard.IO.Xml;
using log4net.Config;
namespace Deltares.Dam.Tools.CsvToDamxConverter
{
internal class Program
{
internal readonly static LogHelper Logger = LogHelper.Create("Main Program.");
const string DamLiveDataFolderName = "DamLiveData";
const string DamXFileExtension = "damx";
const string SensorProfileSheetName = "SensorProfileID";
const string SensorGroupSheetName = "SensorGroupID";
const string SensorSheetName = "SensorID";
const string DikeLineInProfileSheetName = "DikeLineInProfile";
const string IDColumnName = "ID";
const string SensorNameColumnName = "SensorName";
const string PLLineMappingColumnName = "PLLine-Mapping";
const string ProfileColumnName = "Profile";
static bool hasWarnings;
private static string damLiveFolder;
static void Main(string[] args)
{
XmlConfigurator.Configure();
try
{
Run(args);
}
catch (Exception e)
{
Logger.LogFatal("There was an unexpected error. Program terminated", e);
}
//Console.ReadKey();
}
///
/// Runs with the specified arguments.
///
/// The arguments.
///
///
/// An unknown error occurred reading file: + file
///
private static void Run(string[] args)
{
var commandLineArguments = new CommandOptions();
ICommandLineParser parser = new CommandLineParser();
bool success = parser.ParseArguments(args, commandLineArguments);
if (success)
{
var program = new Program();
#if (AUTO_DELETE_EXISTING_OUTPUT_FOLDER)
if (Directory.Exists(commandLineArguments.OuputDataDirectory))
{
const bool recursive = true;
Directory.Delete(commandLineArguments.OuputDataDirectory, recursive);
}
#endif
if (Directory.Exists(commandLineArguments.OuputDataDirectory))
{
string fullPath = Path.GetFullPath(commandLineArguments.OuputDataDirectory);
string msg = string.Format("Output directory {0} already exists. Please delete or use another directory. Program terminated",
fullPath);
throw new DirectoryNotFoundException(msg);
}
string projectName = Path.GetFileNameWithoutExtension(commandLineArguments.DamXFileName);
if (string.IsNullOrWhiteSpace(projectName))
{
const string msg = "Invalid output file name. A valid ." + DamXFileExtension + " file name is required to continue. Program terminated";
throw new InvalidOperationException(msg);
}
string outputDataDirectory = commandLineArguments.OuputDataDirectory;
Directory.CreateDirectory(outputDataDirectory);
string damLiveDataFolderName = string.IsNullOrWhiteSpace(Settings.Default.DamLiveDataFolderName) ? DamLiveDataFolderName : Settings.Default.DamLiveDataFolderName;
damLiveFolder = Path.Combine(commandLineArguments.InputDataDirectory, damLiveDataFolderName);
// Calculation folder (ProjectName.Cacl)
string calcFolder = Path.Combine(commandLineArguments.OuputDataDirectory, projectName + ".Calc");
Directory.CreateDirectory(calcFolder);
// Reading input files
DamProjectData damProjectData = null;
try
{
// old importer
damProjectData = program.LoadDike(commandLineArguments.InputDataDirectory);
Dike dike = damProjectData.WaterBoard.Dikes.First();
// Write output files
CopyGeometryData(projectName, commandLineArguments, dike);
string sensorDataFileName = commandLineArguments.SensorDataFileName;
string excelFile = Path.Combine(commandLineArguments.InputDataDirectory, sensorDataFileName);
if (File.Exists(excelFile))
{
CopySensorDataFromExcel(sensorDataFileName, commandLineArguments, dike);
}
CreateDamXFile(damProjectData, commandLineArguments.OuputDataDirectory, projectName);
if (Directory.Exists(damLiveFolder))
{
string[] files = Directory.GetFiles(damLiveFolder);
foreach (string file in files)
{
string fileName = Path.GetFileName(file);
if (file == null)
{
throw new InvalidOperationException("An unknown error occurred reading file: " + file);
}
File.Copy(file, Path.Combine(outputDataDirectory, fileName));
}
}
Console.WriteLine(hasWarnings
? "Converted the data to a DAMX file, but there are warnings."
: "Conversion of the source data to DAMX file, was successful");
}
finally
{
if (damProjectData != null)
{
damProjectData.Dispose();
}
}
}
else
{
Console.WriteLine();
Console.WriteLine(commandLineArguments.GetUsage());
}
}
///
/// Creates the damx file.
///
/// The dam project data.
/// The output dir.
/// Name of the project.
private static void CreateDamXFile(DamProjectData damProjectData, string outputDir,
string projectName)
{
string fileName = Path.Combine(outputDir, projectName + "." + DamXFileExtension);
fileName = Path.ChangeExtension(fileName, DamXFileExtension);
var xmlSerializer = new XmlSerializer();
xmlSerializer.Serialize(damProjectData, fileName);
}
///
/// Copies the sensor data from excel.
///
/// The source file.
/// The command options.
/// The dike.
///
/// Error reading excel file
/// or
/// The + PLLineMappingColumnName + column has an error. Make sure the cells are not empty and are formatted as text
///
///
private static void CopySensorDataFromExcel(string sourceFile, CommandOptions commandOptions, Dike dike)
{
string[] sheets =
{
SensorProfileSheetName,
SensorGroupSheetName,
SensorSheetName,
DikeLineInProfileSheetName
};
DataSet ds = null;
string fileName = Path.Combine(commandOptions.InputDataDirectory, sourceFile);
string connectionString = string.Format("Provider=Microsoft.Jet.OLEDB.4.0; data source={0}; Extended Properties=Excel 8.0;", fileName);
using (var conn = new OleDbConnection(connectionString))
{
ds = FillDataSet(conn, ds, sheets);
}
if (ds == null)
{
throw new InvalidOperationException("Error reading excel file");
}
EnumerableRowCollection profiles = ds.Tables[SensorProfileSheetName].AsEnumerable();
EnumerableRowCollection groups = ds.Tables[SensorGroupSheetName].AsEnumerable();
EnumerableRowCollection dikeLinePositions = ds.Tables[DikeLineInProfileSheetName].AsEnumerable();
var sensors = new List();
EnumerableRowCollection sensorRows = ds.Tables[SensorSheetName].AsEnumerable();
var count = 0;
foreach (DataRow row in sensorRows)
{
if (row.IsNull(IDColumnName))
{
Logger.LogWarning(string.Format("Error reading row {0} in worksheet {1}. This row will be skipped.", count + 2, SensorSheetName));
continue;
}
count++;
var id = (int) row.Field(IDColumnName);
var name = row.Field(SensorNameColumnName);
var relativeLocation = row.Field("RelativeLocationSensorAlongProfileManual");
var depth = row.Field("DepthSensor");
var enumTypeIgnoreCase = row.Field("SensorType").ToEnumTypeIgnoreCase();
var sensor = new Sensor
{
ID = id,
Name = name,
RelativeLocation = relativeLocation,
Depth = depth,
Type = enumTypeIgnoreCase
//1 255364,05597700000 598646,53034800000
//XRd = 255364.05597700000,
//YRd = 598646.53034800000,
};
try
{
string[] mappings = row.Field(PLLineMappingColumnName)
.Split(new[]
{
';'
}, StringSplitOptions.RemoveEmptyEntries)
.Select(tag => tag.Trim())
//.Where(tag => !string.IsNullOrWhiteSpace(tag))
.ToArray();
foreach (string mapping in mappings)
{
switch (mapping)
{
case "1":
sensor.Add(PLLineType.PL1);
break;
case "2":
sensor.Add(PLLineType.PL2);
break;
case "3":
sensor.Add(PLLineType.PL3);
break;
case "4":
sensor.Add(PLLineType.PL4);
break;
}
}
}
catch (Exception e)
{
throw new InvalidOperationException("The " + PLLineMappingColumnName + " column has an error. Make sure the cells are not empty and are formatted as text", e);
}
Logger.LogInfo(sensor.ToString());
sensors.Add(sensor);
}
Logger.LogInfo(string.Format("Successfully imported {0} sensor(s).", count));
if (!Directory.Exists(damLiveFolder))
{
throw new DirectoryNotFoundException(string.Format("The DAM (CSV) data folder '{0}' doesn't exist. Program terminated.", damLiveFolder));
}
CreateFEWSFilesIfNeeded(commandOptions, sensors);
count = 0;
foreach (Location location in dike.Locations)
{
var repository = new SensorRepository(location);
int id;
if (ParseSensorProfileWorksheet(location, profiles, out id))
{
hasWarnings = true;
continue;
}
// Parsing sensor groups
if (ParseSensorGroups(groups, id, sensors, repository))
{
hasWarnings = true;
continue;
}
count++;
}
Logger.LogInfo(string.Format("Successfully imported {0} sensor groups(s).", count));
Logger.LogInfo(string.Format("Successfully imported {0} sensor profile(s).", count));
}
///
/// Parses the sensor groups.
///
/// The sensor groups.
/// The identifier.
/// The sensors.
/// The repository.
///
///
///
private static bool ParseSensorGroups(IEnumerable sensorGroups, int id, List sensors, SensorRepository repository)
{
const string sensorSelectionColumnName = "SensorSelection";
var currentSensorId = "";
try
{
string[] groupIds = null;
foreach (DataRow sensorGroup in sensorGroups)
{
var idValue = (int) sensorGroup.Field(IDColumnName);
if (idValue != id)
{
continue;
}
var groupIdValue = sensorGroup.Field(sensorSelectionColumnName);
if (groupIdValue == null)
{
Logger.LogWarning(string.Format("Error reading the value from column {0} for group id {1}",
sensorSelectionColumnName, id));
}
else
{
groupIds = groupIdValue.Split(new[]
{
';'
}, StringSplitOptions.RemoveEmptyEntries);
break;
}
}
if (groupIds == null)
{
Logger.LogWarning(string.Format("Couldn't find group id '{0}' in worksheet {1}", id,
SensorGroupSheetName));
return true;
}
foreach (string groupId in groupIds)
{
currentSensorId = groupId;
Sensor single = sensors.Single(s => s.ID.ToString() == groupId);
repository.Add(single);
}
}
catch (InvalidOperationException ioe)
{
if (!string.IsNullOrWhiteSpace(currentSensorId))
{
string message = string.Format("Error finding sensor. The sensor with ID '{0}' was not found in the Excel worksheet '{1}'", currentSensorId, SensorSheetName);
throw new InvalidOperationException(message, ioe);
}
throw ioe;
}
catch (Exception e)
{
throw new InvalidOperationException(
string.Format("An unknown error occurred in column " + sensorSelectionColumnName + " in the Excel sheet " + SensorGroupSheetName +
". Used the following location ID: " + id, e));
}
return false;
}
///
/// Parses the sensor profile worksheet.
///
/// The location.
/// The sensor locations.
/// The identifier.
///
/// An error occurred while parsing row in worksheet + SensorProfileSheetName
private static bool ParseSensorProfileWorksheet(Location location, EnumerableRowCollection sensorLocations, out int id)
{
id = -1;
try
{
Location location1 = location;
var record = sensorLocations
.Where(x => x.Field(ProfileColumnName) == location1.Name)
.Select(x => new
{
ID = GetLocationIDField(x),
GroupID = GetGroupIDField(x),
Alias = GetNameAliasField(x),
InputPL1OuterWaterLevel = GetPL1WaterLevelRiverSide(x),
InputPL1PLLineOffsetBelowDikeTopAtRiver = GetPL1OffsetBelowDikeTopAtRiver(x),
InputPL1PLLineOffsetBelowDikeTopAtPolder = GetPL1OffsetBelowDikeTopAtPolder(x),
InputPL1PLLineOffsetBelowShoulderBaseInside = GetPL1OffsetBelowShoulderBaseInside(x),
InputPL1PLLineOffsetBelowDikeToeAtPolder = GetPL1OffsetBelowDikeToeAtPolder(x),
InputPL1PolderLevel = GetPL1PolderLevelSensorType(x),
InputPL3 = GetInputPl4(x),
InputPL4 = GetInputPl3(x)
}).SingleOrDefault();
if (record == null)
{
Logger.LogWarning(string.Format("Location profile ID '{0}' not found in worksheet {1}. Location profile skipped.",
location.Name, SensorProfileSheetName));
return true;
}
id = record.GroupID;
SensorLocation sensorData = location.SensorLocation;
sensorData.Group.ID = record.GroupID;
sensorData.Alias = record.Alias;
sensorData.SourceTypePl1WaterLevelAtRiver = record.InputPL1OuterWaterLevel;
sensorData.SourceTypePl1PlLineOffsetBelowDikeTopAtRiver = record.InputPL1PLLineOffsetBelowDikeTopAtRiver;
sensorData.SourceTypePl1PlLineOffsetBelowDikeTopAtPolder = record.InputPL1PLLineOffsetBelowDikeTopAtPolder;
sensorData.SourceTypePl1PlLineOffsetBelowShoulderBaseInside = record.InputPL1PLLineOffsetBelowShoulderBaseInside;
sensorData.SourceTypePl1PlLineOffsetBelowDikeToeAtPolder = record.InputPL1PLLineOffsetBelowDikeToeAtPolder;
sensorData.SourceTypePl1WaterLevelAtPolder = record.InputPL1PolderLevel;
sensorData.SourceTypePl3 = record.InputPL3;
sensorData.SourceTypePl4 = record.InputPL4;
}
catch (Exception e)
{
throw new InvalidOperationException("An error occurred while parsing row in worksheet " + SensorProfileSheetName, e);
}
return false;
}
///
/// Creates the fews files if needed.
///
/// The command options.
/// The sensors.
private static void CreateFEWSFilesIfNeeded(CommandOptions commandOptions, List sensors)
{
string[] files = Directory.GetFiles(damLiveFolder);
bool noFilesIncluded = files.Length == 0;
bool inputFileIncluded = files.Any(f => !f.Contains("Parameter"));
if (noFilesIncluded || !inputFileIncluded)
{
string fewsInputFile = Path.Combine(commandOptions.OuputDataDirectory, "live.FEWS.InputFile.xml");
WriteFEWSSensorFiles(new FileInfo(fewsInputFile), sensors);
}
}
///
/// Fills the data set.
///
/// The connection.
/// The ds.
/// The sheets.
///
private static DataSet FillDataSet(OleDbConnection conn, DataSet ds, string[] sheets)
{
conn.Open();
ds = new DataSet();
foreach (string sheet in sheets)
{
// string columns = String.Join(",", columnNames.ToArray());
const string columns = "*";
using (var da = new OleDbDataAdapter(
"SELECT " + columns + " FROM [" + sheet + "$]", conn))
{
var dt = new DataTable(sheet);
da.Fill(dt);
ds.Tables.Add(dt);
}
}
return ds;
}
///
/// Gets the p l1 water level river side.
///
/// The x.
///
private static DataSourceTypeSensors GetPL1WaterLevelRiverSide(DataRow x)
{
DataSourceTypeSensors pl1WaterLevelRiverSide;
pl1WaterLevelRiverSide = x.Field("InputPL1OuterWaterLevel").ToEnumTypeIgnoreCase();
return pl1WaterLevelRiverSide;
}
///
/// Gets the p l1 offset below dike top at river.
///
/// The x.
///
private static DataSourceTypeSensors GetPL1OffsetBelowDikeTopAtRiver(DataRow x)
{
const string columnName = "InputPL1PLLineOffsetBelowDikeTopAtRiver";
DataSourceTypeSensors pl1OffsetBelowDikeTopAtRiver;
pl1OffsetBelowDikeTopAtRiver = x.Field(columnName).ToEnumTypeIgnoreCase();
return pl1OffsetBelowDikeTopAtRiver;
}
///
/// Gets the p l1 offset below dike top at polder.
///
/// The x.
///
private static DataSourceTypeSensors GetPL1OffsetBelowDikeTopAtPolder(DataRow x)
{
const string columnName = "InputPL1PLLineOffsetBelowDikeTopAtPolder";
DataSourceTypeSensors pl1OffsetBelowDikeTopAtPolder;
pl1OffsetBelowDikeTopAtPolder = x.Field(columnName).ToEnumTypeIgnoreCase();
return pl1OffsetBelowDikeTopAtPolder;
}
///
/// Gets the p l1 offset below shoulder base inside.
///
/// The x.
///
private static DataSourceTypeSensors GetPL1OffsetBelowShoulderBaseInside(DataRow x)
{
DataSourceTypeSensors pl1OffsetBelowShoulderBaseInside;
pl1OffsetBelowShoulderBaseInside = x.Field("InputPL1PLLineOffsetBelowShoulderBaseInside").ToEnumTypeIgnoreCase();
return pl1OffsetBelowShoulderBaseInside;
}
///
/// Gets the p l1 offset below dike toe at polder.
///
/// The x.
///
private static DataSourceTypeSensors GetPL1OffsetBelowDikeToeAtPolder(DataRow x)
{
DataSourceTypeSensors pl1OffsetBelowDikeToeAtPolder;
pl1OffsetBelowDikeToeAtPolder = x.Field("InputPL1PLLineOffsetBelowDikeToeAtPolder").ToEnumTypeIgnoreCase();
return pl1OffsetBelowDikeToeAtPolder;
}
///
/// Gets the type of the p l1 polder level sensor.
///
/// The x.
///
private static DataSourceTypeSensors GetPL1PolderLevelSensorType(DataRow x)
{
const string columnName = "InputPL1PolderLevel";
DataSourceTypeSensors pl1PolderLevelSensorType;
pl1PolderLevelSensorType = x.Field(columnName).ToEnumTypeIgnoreCase();
return pl1PolderLevelSensorType;
}
///
/// Gets the input PL4.
///
/// The x.
///
private static DataSourceTypeSensors GetInputPl4(DataRow x)
{
DataSourceTypeSensors dataSourceTypeSensors;
dataSourceTypeSensors = x.Field("InputPL4").ToEnumTypeIgnoreCase();
return dataSourceTypeSensors;
}
///
/// Gets the input PL3.
///
/// The x.
///
private static DataSourceTypeSensors GetInputPl3(DataRow x)
{
DataSourceTypeSensors pl3;
pl3 = x.Field("InputPL3").ToEnumTypeIgnoreCase();
return pl3;
}
///
/// Gets the name alias field.
///
/// The x.
///
/// nameAliasField
private static string GetNameAliasField(DataRow x)
{
var nameAliasField = x.Field("NameAlias");
if (nameAliasField == null)
{
throw new ArgumentNullException("nameAliasField");
}
return nameAliasField;
}
///
/// Gets the group identifier field.
///
/// The x.
///
/// Error parsing value : + x.Field
private static int GetGroupIDField(DataRow x)
{
int groupIdField;
const string columnName = "SensorGroup";
try
{
groupIdField = (int) x.Field(columnName);
}
catch (InvalidCastException)
{
throw new InvalidCastException("Error parsing value : " + x.Field