// Copyright (C) Stichting Deltares 2024. All rights reserved. // // This file is part of the application DAM - UI. // // DAM - UI is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program. If not, see . // // All names, logos, and references to "Deltares" are registered trademarks of // Stichting Deltares and remain full property of Stichting Deltares at all times. // All rights reserved. #if DEBUG #define AUTO_DELETE_EXISTING_OUTPUT_FOLDER #endif using System; using System.Collections.Generic; using System.Data; using System.Data.OleDb; using System.IO; using System.Linq; using CommandLine; using Deltares.Dam.Data; using Deltares.Dam.Data.Sensors; using Deltares.Dam.Tools.CsvToDamxConverter.Properties; using Deltares.Standard.Application; using Deltares.Standard.Extensions; using Deltares.Standard.IO.Xml; using log4net.Config; namespace Deltares.Dam.Tools.CsvToDamxConverter { internal class Program { internal readonly static LogHelper Logger = LogHelper.Create("Main Program."); const string DamLiveDataFolderName = "DamLiveData"; const string DamXFileExtension = "damx"; const string SensorProfileSheetName = "SensorProfileID"; const string SensorGroupSheetName = "SensorGroupID"; const string SensorSheetName = "SensorID"; const string DikeLineInProfileSheetName = "DikeLineInProfile"; const string IDColumnName = "ID"; const string SensorNameColumnName = "SensorName"; const string PLLineMappingColumnName = "PLLine-Mapping"; const string ProfileColumnName = "Profile"; static bool hasWarnings; private static string damLiveFolder; static void Main(string[] args) { XmlConfigurator.Configure(); try { Run(args); } catch (Exception e) { Logger.LogFatal("There was an unexpected error. Program terminated", e); } //Console.ReadKey(); } /// /// Runs with the specified arguments. /// /// The arguments. /// /// /// An unknown error occurred reading file: + file /// private static void Run(string[] args) { var commandLineArguments = new CommandOptions(); ICommandLineParser parser = new CommandLineParser(); bool success = parser.ParseArguments(args, commandLineArguments); if (success) { var program = new Program(); #if (AUTO_DELETE_EXISTING_OUTPUT_FOLDER) if (Directory.Exists(commandLineArguments.OuputDataDirectory)) { const bool recursive = true; Directory.Delete(commandLineArguments.OuputDataDirectory, recursive); } #endif if (Directory.Exists(commandLineArguments.OuputDataDirectory)) { string fullPath = Path.GetFullPath(commandLineArguments.OuputDataDirectory); string msg = string.Format("Output directory {0} already exists. Please delete or use another directory. Program terminated", fullPath); throw new DirectoryNotFoundException(msg); } string projectName = Path.GetFileNameWithoutExtension(commandLineArguments.DamXFileName); if (string.IsNullOrWhiteSpace(projectName)) { const string msg = "Invalid output file name. A valid ." + DamXFileExtension + " file name is required to continue. Program terminated"; throw new InvalidOperationException(msg); } string outputDataDirectory = commandLineArguments.OuputDataDirectory; Directory.CreateDirectory(outputDataDirectory); string damLiveDataFolderName = string.IsNullOrWhiteSpace(Settings.Default.DamLiveDataFolderName) ? DamLiveDataFolderName : Settings.Default.DamLiveDataFolderName; damLiveFolder = Path.Combine(commandLineArguments.InputDataDirectory, damLiveDataFolderName); // Calculation folder (ProjectName.Cacl) string calcFolder = Path.Combine(commandLineArguments.OuputDataDirectory, projectName + ".Calc"); Directory.CreateDirectory(calcFolder); // Reading input files DamProjectData damProjectData = null; try { // old importer damProjectData = program.LoadDike(commandLineArguments.InputDataDirectory); Dike dike = damProjectData.WaterBoard.Dikes.First(); // Write output files CopyGeometryData(projectName, commandLineArguments, dike); string sensorDataFileName = commandLineArguments.SensorDataFileName; string excelFile = Path.Combine(commandLineArguments.InputDataDirectory, sensorDataFileName); if (File.Exists(excelFile)) { CopySensorDataFromExcel(sensorDataFileName, commandLineArguments, dike); } CreateDamXFile(damProjectData, commandLineArguments.OuputDataDirectory, projectName); if (Directory.Exists(damLiveFolder)) { string[] files = Directory.GetFiles(damLiveFolder); foreach (string file in files) { string fileName = Path.GetFileName(file); if (file == null) { throw new InvalidOperationException("An unknown error occurred reading file: " + file); } File.Copy(file, Path.Combine(outputDataDirectory, fileName)); } } Console.WriteLine(hasWarnings ? "Converted the data to a DAMX file, but there are warnings." : "Conversion of the source data to DAMX file, was successful"); } finally { if (damProjectData != null) { damProjectData.Dispose(); } } } else { Console.WriteLine(); Console.WriteLine(commandLineArguments.GetUsage()); } } /// /// Creates the damx file. /// /// The dam project data. /// The output dir. /// Name of the project. private static void CreateDamXFile(DamProjectData damProjectData, string outputDir, string projectName) { string fileName = Path.Combine(outputDir, projectName + "." + DamXFileExtension); fileName = Path.ChangeExtension(fileName, DamXFileExtension); var xmlSerializer = new XmlSerializer(); xmlSerializer.Serialize(damProjectData, fileName); } /// /// Copies the sensor data from excel. /// /// The source file. /// The command options. /// The dike. /// /// Error reading excel file /// or /// The + PLLineMappingColumnName + column has an error. Make sure the cells are not empty and are formatted as text /// /// private static void CopySensorDataFromExcel(string sourceFile, CommandOptions commandOptions, Dike dike) { string[] sheets = { SensorProfileSheetName, SensorGroupSheetName, SensorSheetName, DikeLineInProfileSheetName }; DataSet ds = null; string fileName = Path.Combine(commandOptions.InputDataDirectory, sourceFile); string connectionString = string.Format("Provider=Microsoft.Jet.OLEDB.4.0; data source={0}; Extended Properties=Excel 8.0;", fileName); using (var conn = new OleDbConnection(connectionString)) { ds = FillDataSet(conn, ds, sheets); } if (ds == null) { throw new InvalidOperationException("Error reading excel file"); } EnumerableRowCollection profiles = ds.Tables[SensorProfileSheetName].AsEnumerable(); EnumerableRowCollection groups = ds.Tables[SensorGroupSheetName].AsEnumerable(); EnumerableRowCollection dikeLinePositions = ds.Tables[DikeLineInProfileSheetName].AsEnumerable(); var sensors = new List(); EnumerableRowCollection sensorRows = ds.Tables[SensorSheetName].AsEnumerable(); var count = 0; foreach (DataRow row in sensorRows) { if (row.IsNull(IDColumnName)) { Logger.LogWarning(string.Format("Error reading row {0} in worksheet {1}. This row will be skipped.", count + 2, SensorSheetName)); continue; } count++; var id = (int) row.Field(IDColumnName); var name = row.Field(SensorNameColumnName); var relativeLocation = row.Field("RelativeLocationSensorAlongProfileManual"); var depth = row.Field("DepthSensor"); var enumTypeIgnoreCase = row.Field("SensorType").ToEnumTypeIgnoreCase(); var sensor = new Sensor { ID = id, Name = name, RelativeLocation = relativeLocation, Depth = depth, Type = enumTypeIgnoreCase //1 255364,05597700000 598646,53034800000 //XRd = 255364.05597700000, //YRd = 598646.53034800000, }; try { string[] mappings = row.Field(PLLineMappingColumnName) .Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries) .Select(tag => tag.Trim()) //.Where(tag => !string.IsNullOrWhiteSpace(tag)) .ToArray(); foreach (string mapping in mappings) { switch (mapping) { case "1": sensor.Add(PLLineType.PL1); break; case "2": sensor.Add(PLLineType.PL2); break; case "3": sensor.Add(PLLineType.PL3); break; case "4": sensor.Add(PLLineType.PL4); break; } } } catch (Exception e) { throw new InvalidOperationException("The " + PLLineMappingColumnName + " column has an error. Make sure the cells are not empty and are formatted as text", e); } Logger.LogInfo(sensor.ToString()); sensors.Add(sensor); } Logger.LogInfo(string.Format("Successfully imported {0} sensor(s).", count)); if (!Directory.Exists(damLiveFolder)) { throw new DirectoryNotFoundException(string.Format("The DAM (CSV) data folder '{0}' doesn't exist. Program terminated.", damLiveFolder)); } CreateFEWSFilesIfNeeded(commandOptions, sensors); count = 0; foreach (Location location in dike.Locations) { var repository = new SensorRepository(location); int id; if (ParseSensorProfileWorksheet(location, profiles, out id)) { hasWarnings = true; continue; } // Parsing sensor groups if (ParseSensorGroups(groups, id, sensors, repository)) { hasWarnings = true; continue; } count++; } Logger.LogInfo(string.Format("Successfully imported {0} sensor groups(s).", count)); Logger.LogInfo(string.Format("Successfully imported {0} sensor profile(s).", count)); } /// /// Parses the sensor groups. /// /// The sensor groups. /// The identifier. /// The sensors. /// The repository. /// /// /// private static bool ParseSensorGroups(IEnumerable sensorGroups, int id, List sensors, SensorRepository repository) { const string sensorSelectionColumnName = "SensorSelection"; var currentSensorId = ""; try { string[] groupIds = null; foreach (DataRow sensorGroup in sensorGroups) { var idValue = (int) sensorGroup.Field(IDColumnName); if (idValue != id) { continue; } var groupIdValue = sensorGroup.Field(sensorSelectionColumnName); if (groupIdValue == null) { Logger.LogWarning(string.Format("Error reading the value from column {0} for group id {1}", sensorSelectionColumnName, id)); } else { groupIds = groupIdValue.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries); break; } } if (groupIds == null) { Logger.LogWarning(string.Format("Couldn't find group id '{0}' in worksheet {1}", id, SensorGroupSheetName)); return true; } foreach (string groupId in groupIds) { currentSensorId = groupId; Sensor single = sensors.Single(s => s.ID.ToString() == groupId); repository.Add(single); } } catch (InvalidOperationException ioe) { if (!string.IsNullOrWhiteSpace(currentSensorId)) { string message = string.Format("Error finding sensor. The sensor with ID '{0}' was not found in the Excel worksheet '{1}'", currentSensorId, SensorSheetName); throw new InvalidOperationException(message, ioe); } throw ioe; } catch (Exception e) { throw new InvalidOperationException( string.Format("An unknown error occurred in column " + sensorSelectionColumnName + " in the Excel sheet " + SensorGroupSheetName + ". Used the following location ID: " + id, e)); } return false; } /// /// Parses the sensor profile worksheet. /// /// The location. /// The sensor locations. /// The identifier. /// /// An error occurred while parsing row in worksheet + SensorProfileSheetName private static bool ParseSensorProfileWorksheet(Location location, EnumerableRowCollection sensorLocations, out int id) { id = -1; try { Location location1 = location; var record = sensorLocations .Where(x => x.Field(ProfileColumnName) == location1.Name) .Select(x => new { ID = GetLocationIDField(x), GroupID = GetGroupIDField(x), Alias = GetNameAliasField(x), InputPL1OuterWaterLevel = GetPL1WaterLevelRiverSide(x), InputPL1PLLineOffsetBelowDikeTopAtRiver = GetPL1OffsetBelowDikeTopAtRiver(x), InputPL1PLLineOffsetBelowDikeTopAtPolder = GetPL1OffsetBelowDikeTopAtPolder(x), InputPL1PLLineOffsetBelowShoulderBaseInside = GetPL1OffsetBelowShoulderBaseInside(x), InputPL1PLLineOffsetBelowDikeToeAtPolder = GetPL1OffsetBelowDikeToeAtPolder(x), InputPL1PolderLevel = GetPL1PolderLevelSensorType(x), InputPL3 = GetInputPl4(x), InputPL4 = GetInputPl3(x) }).SingleOrDefault(); if (record == null) { Logger.LogWarning(string.Format("Location profile ID '{0}' not found in worksheet {1}. Location profile skipped.", location.Name, SensorProfileSheetName)); return true; } id = record.GroupID; SensorLocation sensorData = location.SensorLocation; sensorData.Group.ID = record.GroupID; sensorData.Alias = record.Alias; sensorData.SourceTypePl1WaterLevelAtRiver = record.InputPL1OuterWaterLevel; sensorData.SourceTypePl1PlLineOffsetBelowDikeTopAtRiver = record.InputPL1PLLineOffsetBelowDikeTopAtRiver; sensorData.SourceTypePl1PlLineOffsetBelowDikeTopAtPolder = record.InputPL1PLLineOffsetBelowDikeTopAtPolder; sensorData.SourceTypePl1PlLineOffsetBelowShoulderBaseInside = record.InputPL1PLLineOffsetBelowShoulderBaseInside; sensorData.SourceTypePl1PlLineOffsetBelowDikeToeAtPolder = record.InputPL1PLLineOffsetBelowDikeToeAtPolder; sensorData.SourceTypePl1WaterLevelAtPolder = record.InputPL1PolderLevel; sensorData.SourceTypePl3 = record.InputPL3; sensorData.SourceTypePl4 = record.InputPL4; } catch (Exception e) { throw new InvalidOperationException("An error occurred while parsing row in worksheet " + SensorProfileSheetName, e); } return false; } /// /// Creates the fews files if needed. /// /// The command options. /// The sensors. private static void CreateFEWSFilesIfNeeded(CommandOptions commandOptions, List sensors) { string[] files = Directory.GetFiles(damLiveFolder); bool noFilesIncluded = files.Length == 0; bool inputFileIncluded = files.Any(f => !f.Contains("Parameter")); if (noFilesIncluded || !inputFileIncluded) { string fewsInputFile = Path.Combine(commandOptions.OuputDataDirectory, "live.FEWS.InputFile.xml"); WriteFEWSSensorFiles(new FileInfo(fewsInputFile), sensors); } } /// /// Fills the data set. /// /// The connection. /// The ds. /// The sheets. /// private static DataSet FillDataSet(OleDbConnection conn, DataSet ds, string[] sheets) { conn.Open(); ds = new DataSet(); foreach (string sheet in sheets) { // string columns = String.Join(",", columnNames.ToArray()); const string columns = "*"; using (var da = new OleDbDataAdapter( "SELECT " + columns + " FROM [" + sheet + "$]", conn)) { var dt = new DataTable(sheet); da.Fill(dt); ds.Tables.Add(dt); } } return ds; } /// /// Gets the p l1 water level river side. /// /// The x. /// private static DataSourceTypeSensors GetPL1WaterLevelRiverSide(DataRow x) { DataSourceTypeSensors pl1WaterLevelRiverSide; pl1WaterLevelRiverSide = x.Field("InputPL1OuterWaterLevel").ToEnumTypeIgnoreCase(); return pl1WaterLevelRiverSide; } /// /// Gets the p l1 offset below dike top at river. /// /// The x. /// private static DataSourceTypeSensors GetPL1OffsetBelowDikeTopAtRiver(DataRow x) { const string columnName = "InputPL1PLLineOffsetBelowDikeTopAtRiver"; DataSourceTypeSensors pl1OffsetBelowDikeTopAtRiver; pl1OffsetBelowDikeTopAtRiver = x.Field(columnName).ToEnumTypeIgnoreCase(); return pl1OffsetBelowDikeTopAtRiver; } /// /// Gets the p l1 offset below dike top at polder. /// /// The x. /// private static DataSourceTypeSensors GetPL1OffsetBelowDikeTopAtPolder(DataRow x) { const string columnName = "InputPL1PLLineOffsetBelowDikeTopAtPolder"; DataSourceTypeSensors pl1OffsetBelowDikeTopAtPolder; pl1OffsetBelowDikeTopAtPolder = x.Field(columnName).ToEnumTypeIgnoreCase(); return pl1OffsetBelowDikeTopAtPolder; } /// /// Gets the p l1 offset below shoulder base inside. /// /// The x. /// private static DataSourceTypeSensors GetPL1OffsetBelowShoulderBaseInside(DataRow x) { DataSourceTypeSensors pl1OffsetBelowShoulderBaseInside; pl1OffsetBelowShoulderBaseInside = x.Field("InputPL1PLLineOffsetBelowShoulderBaseInside").ToEnumTypeIgnoreCase(); return pl1OffsetBelowShoulderBaseInside; } /// /// Gets the p l1 offset below dike toe at polder. /// /// The x. /// private static DataSourceTypeSensors GetPL1OffsetBelowDikeToeAtPolder(DataRow x) { DataSourceTypeSensors pl1OffsetBelowDikeToeAtPolder; pl1OffsetBelowDikeToeAtPolder = x.Field("InputPL1PLLineOffsetBelowDikeToeAtPolder").ToEnumTypeIgnoreCase(); return pl1OffsetBelowDikeToeAtPolder; } /// /// Gets the type of the p l1 polder level sensor. /// /// The x. /// private static DataSourceTypeSensors GetPL1PolderLevelSensorType(DataRow x) { const string columnName = "InputPL1PolderLevel"; DataSourceTypeSensors pl1PolderLevelSensorType; pl1PolderLevelSensorType = x.Field(columnName).ToEnumTypeIgnoreCase(); return pl1PolderLevelSensorType; } /// /// Gets the input PL4. /// /// The x. /// private static DataSourceTypeSensors GetInputPl4(DataRow x) { DataSourceTypeSensors dataSourceTypeSensors; dataSourceTypeSensors = x.Field("InputPL4").ToEnumTypeIgnoreCase(); return dataSourceTypeSensors; } /// /// Gets the input PL3. /// /// The x. /// private static DataSourceTypeSensors GetInputPl3(DataRow x) { DataSourceTypeSensors pl3; pl3 = x.Field("InputPL3").ToEnumTypeIgnoreCase(); return pl3; } /// /// Gets the name alias field. /// /// The x. /// /// nameAliasField private static string GetNameAliasField(DataRow x) { var nameAliasField = x.Field("NameAlias"); if (nameAliasField == null) { throw new ArgumentNullException("nameAliasField"); } return nameAliasField; } /// /// Gets the group identifier field. /// /// The x. /// /// Error parsing value : + x.Field(columnName) private static int GetGroupIDField(DataRow x) { int groupIdField; const string columnName = "SensorGroup"; try { groupIdField = (int) x.Field(columnName); } catch (InvalidCastException) { throw new InvalidCastException("Error parsing value : " + x.Field(columnName)); } return groupIdField; } /// /// Gets the location identifier field. /// /// The x. /// /// Error parsing value : + x.Field(columnName) + . LocationID should be a numeric value. private static int GetLocationIDField(DataRow x) { int locationIdField; const string columnName = "locationID"; try { locationIdField = (int) x.Field(columnName); } catch (InvalidCastException) { throw new InvalidCastException("Error parsing value : " + x.Field(columnName) + ". LocationID should be a numeric value."); } return locationIdField; } /// /// Copies the geometry data. /// /// Name of the project. /// The command options. /// The dike. private static void CopyGeometryData(string projectName, CommandOptions commandOptions, Dike dike) { const string geomFolderName = "Geometries"; string inputGeomFolder = Path.Combine(commandOptions.InputDataDirectory, geomFolderName); // Calculation folder (ProjectName.Geometries) string destFolderName = projectName + "." + geomFolderName; string geomDestFolder = Path.Combine(commandOptions.OuputDataDirectory, destFolderName); bool hasExternalGeometryFiles = Directory.Exists(inputGeomFolder); if (hasExternalGeometryFiles) { Directory.CreateDirectory(geomDestFolder); // update folder references dike.MapForSoilGeometries2D = destFolderName; foreach (Location location in dike.Locations) { location.MapForSoilGeometries2D = destFolderName; } foreach (string file in Directory.GetFiles(inputGeomFolder)) { string fileName = Path.GetFileName(file); if (fileName != null) { string destFile = Path.Combine(geomDestFolder, fileName); File.Copy(file, destFile); Logger.LogInfo("Copying geometry file: " + destFile); } } } } /// /// Loads the dike. /// /// The project data folder. /// private DamProjectData LoadDike(string projectDataFolder) { var damProjectData = new DamProjectData(); WaterBoard waterBoard = ImportDirector.Construct(projectDataFolder, null); Dike dike = waterBoard.Dikes[0]; try { dike.Validate(); damProjectData.WaterBoard = waterBoard; } catch (Exception) { damProjectData.Dispose(); dike.Dispose(); //dike was not added to Waterboard; Dispose and discard throw; } return damProjectData; } /// /// Writes the fews sensor files. /// /// The file. /// The sensors. private static void WriteFEWSSensorFiles(FileInfo file, IEnumerable sensors) { var collection = new TimeSerieCollection(); foreach (Sensor sensor in sensors) { TimeSerie series = collection.AddNewSeries(sensor.Name); series.TimeStep.Multiplier = 3600; series.TimeStep.Unit = TimeStepUnit.Second; series.StartDateTime = new DateTime(2012, 12, 31); series.EndDateTime = new DateTime(2012, 12, 31, 1, 0, 0); } collection.Save(file.FullName); } } }