// Copyright (C) Stichting Deltares 2024. All rights reserved.
//
// This file is part of the application DAM - UI.
//
// DAM - UI is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see .
//
// All names, logos, and references to "Deltares" are registered trademarks of
// Stichting Deltares and remain full property of Stichting Deltares at all times.
// All rights reserved.
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Deltares.Dam.Data;
using Deltares.Dam.Data.DataPlugins;
using Deltares.Dam.Data.DataPlugins.Configuration;
using Deltares.Geometry;
using Deltares.Geotechnics.Soils;
using Deltares.Geotechnics.SurfaceLines;
using Deltares.Standard.Language;
using Deltares.Standard.Logging;
using NUnit.Framework;
namespace Deltares.Dam.Tests;
[TestFixture]
public class WaterBoardImporterTests
{
private readonly string directoryWithCsvFiles = Path.Combine(Directory.GetCurrentDirectory(), @"TestData\GrootSallandBinnenwaarts\");
private DataPluginImporter dataPluginImporter;
private DataSourceContainer dataSourceContainer;
[Test]
public void WaterBoardImportedWithCsvFilesHasValidData()
{
const string soilKleiDuin = "klei duin";
const string soilProfileId = "10Z_275_STBI_p";
const string segmentId = "segment_dijkring10_dwp02_4";
const string surfacelineId = "dijkring10_dwp40_0";
const string locationId = "dwp04_1";
const double cTolerance = 0.00001;
const double cToleranceCoordinate = 0.00001;
SetupForCsvFilesImport();
using WaterBoard waterBoard = WaterBoardImporter.ImportAllData("", dataSourceContainer,
DamProjectType.Calamity, null, out List _);
// Check Dike
Assert.That(waterBoard.Dikes.Count, Is.EqualTo(1));
Dike dike = waterBoard.Dikes[0];
//Check Soils
// ToDO For now expect 0 as the soildb for this test has not yet been replaced with a soils.csv
Assert.That(dike.SoilList.Soils.Count, Is.EqualTo(0));
// Assert.AreEqual(47, dike.SoilList.Soils.Count);
// Soil soil = dike.SoilList.Soils[dike.SoilList.GetSoilIndexByName(soilKleiDuin)];
// Assert.AreEqual(15.7, soil.AbovePhreaticLevel, cTolerance);
// Check Soilprofiles
Assert.That(dike.SoilProfiles.Count, Is.EqualTo(26));
SoilProfile1D soilProfile = dike.SoilProfiles.First(x => x.Name.Equals(soilProfileId));
Assert.That(soilProfile.Layers[0].Soil.Name, Is.EqualTo(soilKleiDuin));
// Check Segments
Assert.That(waterBoard.Segments.Count, Is.EqualTo(24));
Segment segment = waterBoard.Segments.First(x => x.Name.Equals(segmentId));
Assert.That(segment.SoilProfileProbabilities[0].SoilGeometryName, Is.EqualTo("10Y_024_STBI.sti"));
Assert.That(segment.SoilProfileProbabilities[0].Probability, Is.EqualTo(100.0).Within(cTolerance));
Assert.That(segment.SoilProfileProbabilities[0].SegmentFailureMechanismType
.Equals(FailureMechanismSystemType.StabilityInside), Is.True);
// Check segment with stability and piping
segment = waterBoard.Segments.FirstOrDefault(x => x.Name.Equals("segment_dijkring10_dwp02_4"));
Assert.That(segment.SoilProfileProbabilities[0].SoilGeometryName, Is.EqualTo("10Y_024_STBI.sti"));
Assert.That(segment.SoilProfileProbabilities[0].Probability, Is.EqualTo(100.0).Within(cTolerance));
Assert.That(segment.SoilProfileProbabilities[0].SegmentFailureMechanismType
.Equals(FailureMechanismSystemType.StabilityInside), Is.True);
Assert.That(segment.SoilProfileProbabilities[1].SoilGeometryName, Is.EqualTo("10Y_024_STBI.sti"));
Assert.That(segment.SoilProfileProbabilities[1].Probability, Is.EqualTo(100.0).Within(cTolerance));
Assert.That(segment.SoilProfileProbabilities[1].SegmentFailureMechanismType
.Equals(FailureMechanismSystemType.Piping), Is.True);
// Check surfacelines
Assert.That(dike.SurfaceLines2.Count, Is.EqualTo(13));
SurfaceLine2 surfaceline = dike.SurfaceLines2.First(x => x.Name.Equals(surfacelineId));
Assert.That(surfaceline.Geometry.Points.Any(), Is.True);
Assert.That(surfaceline.Geometry.Points[0].X, Is.EqualTo(0.0).Within(cToleranceCoordinate));
Assert.That(surfaceline.Geometry.Points[0].Y, Is.EqualTo(0.0).Within(cToleranceCoordinate));
Assert.That(surfaceline.Geometry.Points[0].Z, Is.EqualTo(-0.48).Within(cToleranceCoordinate));
int lastPointIndex = surfaceline.Geometry.Count - 1;
Assert.That(surfaceline.Geometry.Points[lastPointIndex].X, Is.EqualTo(71.25).Within(cToleranceCoordinate));
Assert.That(surfaceline.Geometry.Points[lastPointIndex].Y, Is.EqualTo(0.0).Within(cToleranceCoordinate));
Assert.That(surfaceline.Geometry.Points[lastPointIndex].Z, Is.EqualTo(-0.36).Within(cToleranceCoordinate));
GeometryPoint point = surfaceline.CharacteristicPoints.GetGeometryPoint(
CharacteristicPointType.SurfaceLevelInside);
Assert.That(point.X, Is.EqualTo(71.25).Within(cToleranceCoordinate));
Assert.That(point.Y, Is.EqualTo(0.0).Within(cToleranceCoordinate));
Assert.That(point.Z, Is.EqualTo(-0.36).Within(cToleranceCoordinate));
// Check locations
Assert.That(dike.Locations.Count, Is.EqualTo(13));
Location location = dike.Locations.First(x => x.Name.Equals(locationId));
// Assert.AreEqual(0.25, location.MinimalCircleDepth, cTolerance); // is default from modelparameters
Assert.That(location.Segment.Name, Is.EqualTo("segment_dijkring10_dwp04_1"));
Assert.That(location.SurfaceLine2.Name, Is.EqualTo("dijkring10_dwp04_1"));
dike.Validate();
}
[Test]
public void ImportWaterBoardOnlyCsvFiles1D()
{
const double cTolerance = 0.0001;
const string definitionFilename = @".\TestData\CSVData\Full1DProject\Import.defx";
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
IList dikeList = WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer,
DamProjectType.Calamity, out _).ToList();
using WaterBoard waterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder,
dataSourceContainer, dikeList,
DamProjectType.Calamity, null,
out List _);
// Check Dike
Assert.That(waterBoard.Dikes, Has.Count.EqualTo(1));
Dike dike = waterBoard.Dikes[0];
//Check locations
Assert.That(dike.Locations, Has.Count.EqualTo(2));
// Check Soilprofiles
Assert.That(dike.SoilProfiles, Has.Count.EqualTo(23));
SoilProfile1D soilProfile = dike.SoilProfiles.First(x => x.Name.Equals("25_2_1_bz_4"));
Assert.That(soilProfile.Layers[0].Soil.Name, Is.EqualTo("kade"));
//Check Soils
Assert.That(dike.SoilList.Soils, Has.Count.EqualTo(51));
Soil soil = dike.SoilList.Soils[dike.SoilList.GetSoilIndexByName("kade")];
Assert.That(soil.AbovePhreaticLevel, Is.EqualTo(17.0).Within(cTolerance));
//Check Soil of shear strength model SigmaTauTable with sigma-tau table
soil = dike.SoilList.Soils[dike.SoilList.GetSoilIndexByName("CCC")];
Assert.Multiple(() =>
{
Assert.That(soil.ShearStrengthModel, Is.EqualTo(ShearStrengthModel.StressTable));
Assert.That(soil.StressTable.Name, Is.EqualTo("CurveKlei"));
Assert.That(soil.StressTable.SigmaTaus, Has.Count.EqualTo(5));
});
Assert.Multiple(() =>
{
Assert.That(soil.StressTable.SigmaTaus[0].Sigma, Is.EqualTo(0.0).Within(cTolerance));
Assert.That(soil.StressTable.SigmaTaus[0].Tau, Is.EqualTo(2.05).Within(cTolerance));
Assert.That(soil.StressTable.SigmaTaus[4].Sigma, Is.EqualTo(110.5).Within(cTolerance));
Assert.That(soil.StressTable.SigmaTaus[4].Tau, Is.EqualTo(44.6).Within(cTolerance));
});
//Check Soil of shear strength model MohrCoulomb with sigma-tau table
soil = dike.SoilList.Soils[dike.SoilList.GetSoilIndexByName("BSS")];
Assert.Multiple(() =>
{
Assert.That(soil.ShearStrengthModel, Is.EqualTo(ShearStrengthModel.CPhi));
Assert.That(soil.StressTable.Name, Is.EqualTo("CurveZand"));
Assert.That(soil.StressTable.SigmaTaus, Has.Count.EqualTo(2));
});
Assert.Multiple(() =>
{
Assert.That(soil.StressTable.SigmaTaus[0].Sigma, Is.EqualTo(0.0).Within(cTolerance));
Assert.That(soil.StressTable.SigmaTaus[0].Tau, Is.EqualTo(0.0).Within(cTolerance));
Assert.That(soil.StressTable.SigmaTaus[1].Sigma, Is.EqualTo(200.0).Within(cTolerance));
Assert.That(soil.StressTable.SigmaTaus[1].Tau, Is.EqualTo(129.88).Within(cTolerance));
});
// Check Segments
Assert.That(waterBoard.Segments, Has.Count.EqualTo(2));
Segment segment = waterBoard.Segments.First(x => x.Name.Equals("106"));
Assert.Multiple(() =>
{
Assert.That(segment.SoilProfileProbabilities[0].SoilGeometryName, Is.EqualTo("25_2_1_bz_1"));
Assert.That(segment.SoilProfileProbabilities[0].Probability, Is.EqualTo(35.0).Within(cTolerance));
Assert.That(segment.SoilProfileProbabilities[0].SegmentFailureMechanismType,
Is.EqualTo(FailureMechanismSystemType.StabilityInside));
});
}
[Test]
public void ImportWaterBoardCsvFiles1DMissingSigmaTauCurve()
{
const string definitionFilename = @".\TestData\CSVData\Full1DProjectMissingSigmaTauTable\Import.defx";
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
List logMessages;
IList dikeList = WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer,
DamProjectType.Calamity, out _).ToList();
using WaterBoard waterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder,
dataSourceContainer, dikeList,
DamProjectType.Calamity, null,
out logMessages);
Assert.That(logMessages, Has.Count.GreaterThan(0));
Assert.Multiple(() =>
{
Assert.That(logMessages[0].Message, Contains.Substring("Sigma-Tau curve table Missing not found in CSV file for soil CCC (or sigmataucurves.csv missing)."));
Assert.That(logMessages[1].Message, Contains.Substring("Sigma-Tau curve table MissingNotUsed not found in CSV file for soil ETL (or sigmataucurves.csv missing)."));
});
// Check Dike
Assert.That(waterBoard.Dikes, Has.Count.EqualTo(1));
//Check locations
Assert.That(waterBoard.Dikes[0].Locations, Has.Count.EqualTo(0));
}
[Test]
public void ImportWaterBoardCsvFiles1DMissingSigmaTauCurveCsvFile()
{
const string definitionFilename = @".\TestData\CSVData\Full1DProjectMissingSigmaTauCsvFile\Import.defx";
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
List logMessages;
IList dikeList = WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer,
DamProjectType.Calamity, out _).ToList();
using WaterBoard waterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder,
dataSourceContainer, dikeList,
DamProjectType.Calamity, null,
out logMessages);
Assert.That(logMessages, Has.Count.GreaterThan(0));
Assert.Multiple(() =>
{
Assert.That(logMessages[0].Message, Contains.Substring("Sigma-Tau curve table Missing not found in CSV file for soil CCC (or sigmataucurves.csv missing)."));
// Check Dike
Assert.That(waterBoard.Dikes, Has.Count.EqualTo(1));
//Check locations
Assert.That(waterBoard.Dikes[0].Locations, Has.Count.EqualTo(0));
});
}
[Test]
public void ImportWaterBoardOnlyCsvFiles2D()
{
const string definitionFilename = @".\TestData\CSVData\Full2DProject\Import.defx";
const string projectFilename = @".\TestData\TestWithGeometries2D.damx";
using (var damProject = new DamProject())
{
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
IList dikeList = WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer,
DamProjectType.Calamity, out _).ToList();
damProject.DamProjectData.WaterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder,
dataSourceContainer,
dikeList, DamProjectType.Calamity, null, out List _);
damProject.AssignGeometry2DMapnameIfNotAssigned(Path.Combine(damImportFolder, dataSourceContainer.MapSoilProfile2D));
// Save the data
damProject.SaveXMLProject(projectFilename, damProject);
// Check Dike
Assert.That(damProject.DamProjectData.WaterBoard.Dikes.Count, Is.EqualTo(1));
Dike dike = damProject.DamProjectData.WaterBoard.Dikes[0];
//Check locations
Assert.That(dike.Locations.Count, Is.EqualTo(17));
}
}
[Test]
public void ImportWaterBoardOnlyWithoutPl3AndPl4CsvFiles()
{
const string definitionFilename = @".\TestData\CSVData\DataWithoutPl3AndPl4\Import.defx";
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
IList dikeList = WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer,
DamProjectType.Calamity, out _).ToList();
using WaterBoard waterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder,
dataSourceContainer, dikeList,
DamProjectType.Calamity, null, out List _);
// Check Dike
Assert.That(waterBoard.Dikes.Count, Is.EqualTo(1));
Dike dike = waterBoard.Dikes[0];
//Check locations
Assert.That(dike.Locations.Count, Is.EqualTo(2));
}
[Test]
public void ImportWaterBoardOnlyCsvFiles1DRelativeSoilProfiles()
{
const double cTolerance = 0.0001;
const string definitionFilename = @".\TestData\CSVData\Full1DProject\Import relative soilprofiles.defx";
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
IList dikeList = WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer, DamProjectType.Calamity,
out _).ToList();
using (WaterBoard waterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder, dataSourceContainer,
dikeList, DamProjectType.Calamity, null, out List _))
using (WaterBoard referenceWaterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder,
dataSourceContainer,
dikeList, DamProjectType.Calamity, null, out List _))
{
// If profiles are defined as relative profiles, new absolute profiles will be generated for each location
if (dataSourceContainer.IsImportAsRelativeProfiles)
{
WaterBoardPostProcessRelativeProfiles.CreateAbsoluteProfiles(waterBoard,
dataSourceContainer.SoilProfileCharacteristicPointReference);
}
// Check Dike
Assert.That(waterBoard.Dikes.Count, Is.EqualTo(1));
Dike dike = waterBoard.Dikes[0];
//Check locations
Assert.That(dike.Locations.Count, Is.EqualTo(2));
// Check Soilprofiles
// Count is 43: 23 (imported) + 7 (generated for first location) + 13 (generated for second location)
Assert.That(dike.SoilProfiles.Count, Is.EqualTo(43));
SoilProfile1D soilProfile = dike.SoilProfiles.First(x => x.Name.Equals("25_2_1_bz_4"));
Assert.That(soilProfile.Layers[0].Soil.Name, Is.EqualTo("kade"));
Assert.That(soilProfile.TopLevel, Is.EqualTo(60.0));
// Check if all profiles are made absolute from relative
foreach (Location location in waterBoard.Locations)
foreach (SoilGeometryProbability soilProfileProbability in location.Segment.SoilProfileProbabilities)
{
GeometryPoint dikeToeAtPolder = location.SurfaceLine2.CharacteristicPoints.GetGeometryPoint(CharacteristicPointType
.DikeToeAtPolder);
Assert.That(dikeToeAtPolder.Z, Is.EqualTo(soilProfileProbability.SoilProfile.Layers[1].TopLevel).Within(cTolerance));
}
// Compare reference waterboard to converted waterboard
foreach (Location location in waterBoard.Locations)
{
Location referenceLocation = referenceWaterBoard.Locations.Find(l => l.Name.Equals(location.Name));
var soilProbabilityIndex = 0;
foreach (SoilGeometryProbability referenceSoilProfileProbability in referenceLocation.Segment.SoilProfileProbabilities)
{
soilProbabilityIndex++;
SoilGeometryProbability soilProfileProbability =
location.Segment.SoilProfileProbabilities.Find(
s => s.SoilProfile.Name.Equals(string.Format("{0}-{1}-{2}",
referenceSoilProfileProbability.SoilProfile.Name, location.Name, soilProbabilityIndex)));
for (var layerIndex = 0;
layerIndex < soilProfileProbability.SoilProfile.LayerCount;
layerIndex++)
{
// As the moved profile has a new top layer, make sure to compare the proper layers
if (layerIndex > 0)
{
Assert.That(
soilProfileProbability.SoilProfile.Layers[layerIndex].Height,
Is.EqualTo(referenceSoilProfileProbability.SoilProfile.Layers[layerIndex - 1].Height).Within(cTolerance),
string.Format(
"Location {0} Segment {1} soilProfile {2} Layer {3} ({4}/{5}), Org ({6}/{7})",
location.Name, location.Segment.Name, soilProfileProbability.SoilProfile.Name,
soilProfileProbability.SoilProfile.Layers[layerIndex],
soilProfileProbability.SoilProfile.Layers[layerIndex].TopLevel,
soilProfileProbability.SoilProfile.Layers[layerIndex].BottomLevel,
referenceSoilProfileProbability.SoilProfile.Layers[layerIndex - 1].TopLevel,
referenceSoilProfileProbability.SoilProfile.Layers[layerIndex - 1].BottomLevel));
}
}
}
}
}
}
[Test]
public void ThrowsExceptionWhenOpeningDatasourceFileWithSoilBaseDefinition()
{
const string definitionFilename = @".\TestData\CSVData\Full1DProject\ImportMSoilBase.defx";
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
LanguageType oldLanguage = LocalizationManager.CurrentLanguage;
try
{
LocalizationManager.CurrentLanguage = LanguageType.English;
Assert.That(() => WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer, DamProjectType.Calamity, out _),
Throws.InstanceOf().With.Message.EqualTo("DataLocation is empty. Perhaps the data source type is not supported."));
}
finally
{
LocalizationManager.CurrentLanguage = oldLanguage;
}
}
[Test]
public void IsZoneDataReadCorrectlyFromCsvFiles()
{
const double cTolerance = 0.0001;
const string definitionFilename = @".\TestData\CSVData\ProjectZoneType\Import.defx";
dataSourceContainer = DataSourceContainer.Deserialize(definitionFilename);
string damImportFolder = Path.GetDirectoryName(definitionFilename);
IList dikeList =
WaterBoardImporter.ImportDikeRingIds(damImportFolder, dataSourceContainer, DamProjectType.Calamity,
out _).ToList();
using (WaterBoard waterBoard = WaterBoardImporter.ImportDataForDikeRings("", damImportFolder, dataSourceContainer,
dikeList, DamProjectType.Calamity, null, out List _))
{
// Check Dike
Assert.That(waterBoard.Dikes.Count, Is.EqualTo(1));
Dike dike = waterBoard.Dikes[0];
//Check locations
Assert.That(dike.Locations.Count, Is.EqualTo(2));
Assert.That(dike.Locations[0].StabilityZoneType, Is.EqualTo(MStabZonesType.NoZones));
Assert.That(dike.Locations[0].ForbiddenZoneFactor, Is.EqualTo(0.5).Within(cTolerance));
Assert.That(dike.Locations[1].StabilityZoneType, Is.EqualTo(MStabZonesType.ForbiddenZone));
Assert.That(dike.Locations[1].ForbiddenZoneFactor, Is.EqualTo(0.9).Within(cTolerance));
}
}
private void SetupForCsvFilesImport()
{
var srcDataSources = new List
{
new()
{
DataSourceType = DataSourceType.CsvFiles,
DataLocation = directoryWithCsvFiles
}
};
dataPluginImporter = new DataPluginImporter();
dataPluginImporter.SetDataSources("", srcDataSources);
dataSourceContainer = new DataSourceContainer
{
DataSourceList = srcDataSources
};
}
}